Add git to webapp base container.

Merge pull request 'Fix mainnet laconic deploy setup' (#850 ) from dboreham/fix-mainnet-laconic-init into main
Reviewed-on: cerc-io/stack-orchestrator#850
2024-06-24 23:57:15 -05:00 · 2024-06-22 01:25:07 +00:00 · 2024-06-21 19:24:33 -06:00 · 2024-06-20 09:27:23 +00:00 · 2024-06-19 10:40:54 +00:00 · 2024-06-19 06:41:58 +00:00
43 changed files with 2991 additions and 445 deletions
@@ -4,3 +4,4 @@ Trigger
 Trigger
 Trigger
 Trigger
+Trigger
@@ -18,6 +18,8 @@ services:
      "/update-grafana-alerts-config.sh && /run.sh"
    ports:
      - "3000"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
    healthcheck:
      test: ["CMD", "nc", "-vz", "localhost", "3000"]
      interval: 30s
@@ -22,6 +22,7 @@ services:
      GRAPH_ETHEREUM_JSON_RPC_TIMEOUT: ${GRAPH_ETHEREUM_JSON_RPC_TIMEOUT:-180}
      GRAPH_ETHEREUM_REQUEST_RETRIES: ${GRAPH_ETHEREUM_REQUEST_RETRIES:-10}
      GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE: ${GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE:-2000}
+      GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS: ${GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS:-1000}
    entrypoint: ["bash", "-c"]
    # Wait for ETH RPC endpoint to be up when running with fixturenet-lotus
    command: |
@@ -31,6 +32,7 @@ services:
      - "8001"
      - "8020"
      - "8030"
+      - "8040"
    healthcheck:
      test: ["CMD", "nc", "-vz", "localhost", "8020"]
      interval: 30s
@@ -28,15 +28,37 @@ services:
    extra_hosts:
      - "host.docker.internal:host-gateway"

-  chain-head-exporter:
+  ethereum-chain-head-exporter:
    image: cerc/watcher-ts:local
    restart: always
    working_dir: /app/packages/cli
    environment:
-      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
-      FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT}
+      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT:-https://mainnet.infura.io/v3}
      ETH_RPC_API_KEY: ${CERC_INFURA_KEY}
-      PORT: ${CERC_METRICS_PORT}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  filecoin-chain-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT:-https://api.node.glif.io/rpc/v1}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  graph-node-upstream-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${GRAPH_NODE_RPC_ENDPOINT}
    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
    ports:
      - '5000'
@@ -1,13 +0,0 @@
-services:
-  snowballtools-base-backend:
-    image: cerc/snowballtools-base-backend:local
-    restart: always
-    volumes:
-      - data:/data
-      - config:/config:ro
-    ports:
-      - 8000
-
-volumes:
-  data:
-  config:
@@ -60,6 +60,7 @@ services:
    volumes:
      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-ajna/start-server.sh:/app/start-server.sh
+      - ajna_watcher_gql_logs_data:/app/gql-logs
    ports:
      - "3008"
      - "9001"
@@ -74,3 +75,4 @@ services:

 volumes:
  ajna_watcher_db_data:
+  ajna_watcher_gql_logs_data:
@@ -74,8 +74,10 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/azimuth-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/azimuth-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/azimuth-watcher/start-server.sh
+      - azimuth_watcher_gql_logs_data:/app/packages/azimuth-watcher/gql-logs
    ports:
      - "3001"
+      - "9001"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3001"]
      interval: 20s
@@ -135,8 +137,10 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/censures-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/censures-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/censures-watcher/start-server.sh
+      - censures_watcher_gql_logs_data:/app/packages/censures-watcher/gql-logs
    ports:
      - "3002"
+      - "9003"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3002"]
      interval: 20s
@@ -196,8 +200,10 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/claims-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/claims-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/claims-watcher/start-server.sh
+      - claims_watcher_gql_logs_data:/app/packages/claims-watcher/gql-logs
    ports:
      - "3003"
+      - "9005"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3003"]
      interval: 20s
@@ -257,8 +263,10 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/conditional-star-release-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/conditional-star-release-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/conditional-star-release-watcher/start-server.sh
+      - conditional_star_release_watcher_gql_logs_data:/app/packages/conditional-star-release-watcher/gql-logs
    ports:
      - "3004"
+      - "9007"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3004"]
      interval: 20s
@@ -318,8 +326,10 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/delegated-sending-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/delegated-sending-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/delegated-sending-watcher/start-server.sh
+      - delegated_sending_watcher_gql_logs_data:/app/packages/delegated-sending-watcher/gql-logs
    ports:
      - "3005"
+      - "9009"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3005"]
      interval: 20s
@@ -379,8 +389,10 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/ecliptic-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/ecliptic-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/ecliptic-watcher/start-server.sh
+      - ecliptic_watcher_gql_logs_data:/app/packages/ecliptic-watcher/gql-logs
    ports:
      - "3006"
+      - "9011"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3006"]
      interval: 20s
@@ -440,8 +452,10 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/linear-star-release-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/linear-star-release-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/linear-star-release-watcher/start-server.sh
+      - linear_star_release_watcher_gql_logs_data:/app/packages/linear-star-release-watcher/gql-logs
    ports:
      - "3007"
+      - "9013"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3007"]
      interval: 20s
@@ -501,8 +515,10 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/polls-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/polls-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/polls-watcher/start-server.sh
+      - polls_watcher_gql_logs_data:/app/packages/polls-watcher/gql-logs
    ports:
      - "3008"
+      - "9015"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
@@ -552,3 +568,11 @@ services:

 volumes:
  watcher_db_data:
+  azimuth_watcher_gql_logs_data:
+  censures_watcher_gql_logs_data:
+  claims_watcher_gql_logs_data:
+  conditional_star_release_watcher_gql_logs_data:
+  delegated_sending_watcher_gql_logs_data:
+  ecliptic_watcher_gql_logs_data:
+  linear_star_release_watcher_gql_logs_data:
+  polls_watcher_gql_logs_data:
@@ -60,6 +60,7 @@ services:
    volumes:
      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh
+      - merkl_sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
    ports:
      - "127.0.0.1:3007:3008"
      - "9003:9001"
@@ -74,3 +75,4 @@ services:

 volumes:
  merkl_sushiswap_v3_watcher_db_data:
+  merkl_sushiswap_v3_watcher_gql_logs_data:
@@ -60,6 +60,7 @@ services:
    volumes:
      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh
+      - sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
    ports:
      - "127.0.0.1:3008:3008"
      - "9001:9001"
@@ -74,3 +75,4 @@ services:

 volumes:
  sushiswap_v3_watcher_db_data:
+  sushiswap_v3_watcher_gql_logs_data:
@@ -0,0 +1,20 @@
+apiVersion: 1
+
+datasources:
+  - name: Graph Node Postgres
+    type: postgres
+    jsonData:
+      database: graph-node
+      sslmode: 'disable'
+      maxOpenConns: 100
+      maxIdleConns: 100
+      maxIdleConnsAuto: true
+      connMaxLifetime: 14400
+      postgresVersion: 1411 # 903=9.3, 1000=10, 1411=14.11
+      timescaledb: false
+    user: graph-node
+    # # Add URL for graph-node database
+    # url: graph-node-db:5432
+    # # Set password for graph-node database
+    # secureJsonData:
+    #   password: 'password'
@@ -45,7 +45,18 @@ scrape_configs:
    metrics_path: /metrics
    scheme: http
    static_configs:
-      - targets: ['chain-head-exporter:5000']
+      - targets: ['ethereum-chain-head-exporter:5000']
+        labels:
+          instance: 'external'
+          chain: 'ethereum'
+      - targets: ['filecoin-chain-head-exporter:5000']
+        labels:
+          instance: 'external'
+          chain: 'filecoin'
+      - targets: ['graph-node-upstream-head-exporter:5000']
+        labels:
+          instance: 'graph-node'
+          chain: 'filecoin'

  - job_name: 'postgres'
    scrape_interval: 30s
@@ -74,3 +85,11 @@ scrape_configs:
      # - targets: ['example-host:1317']
    params:
      format: ['prometheus']
+
+  - job_name: graph-node
+    metrics_path: /metrics
+    scrape_interval: 30s
+    scheme: http
+    static_configs:
+      # Add graph-node targets to be monitored below
+      # - targets: ['graph-node:8040']
@@ -4,4 +4,6 @@ echo Using CERC_GRAFANA_ALERTS_SUBGRAPH_IDS ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}

 # Replace subgraph ids in subgraph alerting config
 # Note: Requires the grafana container to be run with user root
-sed -i "s/REPLACE_WITH_SUBGRAPH_IDS/$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS/g" /etc/grafana/provisioning/alerting/subgraph-alert-rules.yml
+if [ -n "$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS" ]; then
+  sed -i "s/REPLACE_WITH_SUBGRAPH_IDS/$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS/g" /etc/grafana/provisioning/alerting/subgraph-alert-rules.yml
+fi
@@ -24,7 +24,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -100,7 +100,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -176,7 +176,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -252,7 +252,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -328,7 +328,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -404,7 +404,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -480,7 +480,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -556,7 +556,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -634,7 +634,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -710,7 +710,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -788,7 +788,85 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      
+      # Secured Finance
+      - uid: secured_finance_diff_external
+        title: secured_finance_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="secured-finance", instance="secured-finance", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -2,7 +2,6 @@
  host = "0.0.0.0"
  port = 3008
  kind = "active"
-  gqlPath = "/"

  # Checkpointing state.
  checkpointing = true
@@ -22,23 +21,30 @@
  # Interval in number of blocks at which to clear entities cache.
  clearEntitiesCacheInterval = 1000

-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
  rpcSupportsBlockHashParam = false

-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"

-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000

-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day

 [metrics]
  host = "0.0.0.0"
@@ -85,6 +91,9 @@
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000

+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -1,6 +1,7 @@
 [server]
  host = "0.0.0.0"
-  maxSimultaneousRequests = -1
+  [server.gql]
+    maxSimultaneousRequests = -1

 [metrics]
  host = "0.0.0.0"
@@ -2,7 +2,6 @@
  host = "0.0.0.0"
  port = 3008
  kind = "active"
-  gqlPath = '/'

  # Checkpointing state.
  checkpointing = true
@@ -22,23 +21,30 @@
  # Interval in number of blocks at which to clear entities cache.
  clearEntitiesCacheInterval = 1000

-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
  rpcSupportsBlockHashParam = false

-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"

-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000

-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day

 [metrics]
  host = "0.0.0.0"
@@ -85,6 +91,9 @@
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000

+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -2,7 +2,6 @@
  host = "0.0.0.0"
  port = 3008
  kind = "active"
-  gqlPath = "/"

  # Checkpointing state.
  checkpointing = true
@@ -22,23 +21,30 @@
  # Interval in number of blocks at which to clear entities cache.
  clearEntitiesCacheInterval = 1000

-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
  rpcSupportsBlockHashParam = false

-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"

-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000

-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day

 [metrics]
  host = "0.0.0.0"
@@ -85,6 +91,9 @@
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000

+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -1,6 +0,0 @@
-FROM cerc/snowballtools-base-backend-base:local
-
-WORKDIR /app/packages/backend
-COPY run.sh .
-
-ENTRYPOINT ["./run.sh"]
@@ -1,26 +0,0 @@
-FROM ubuntu:22.04 as builder
-
-RUN apt update && \
-      apt install -y --no-install-recommends --no-install-suggests \
-      ca-certificates curl gnupg
-
-# Node
-ARG NODE_MAJOR=20
-RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
-      echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
-      apt update && apt install -y nodejs
-
-# npm setup
-RUN npm config set @cerc-io:registry https://git.vdb.to/api/packages/cerc-io/npm/ && npm install -g yarn
-
-COPY . /app/
-WORKDIR /app/
-
-RUN find . -name 'node_modules' | xargs -n1 rm -rf
-RUN yarn && yarn build --ignore frontend
-
-FROM cerc/webapp-base:local
-
-COPY --from=builder /app /app
-
-WORKDIR /app/packages/backend
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-# Build cerc/webapp-deployer-backend
-
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-
-# See: https://stackoverflow.com/a/246128/1701505
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-docker build -t cerc/snowballtools-base-backend-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile-base ${CERC_REPO_BASE_DIR}/snowballtools-base
-docker build -t cerc/snowballtools-base-backend:local ${build_command_args} ${SCRIPT_DIR} 
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-
-LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
-if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
-  if [ -f "/config/laconic-hosted-config.yml" ]; then
-    LACONIC_HOSTED_CONFIG_FILE="/config/laconic-hosted-config.yml"
-  elif [ -f "/config/config.yml" ]; then
-    LACONIC_HOSTED_CONFIG_FILE="/config/config.yml"
-  fi
-fi
-
-if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
-  /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "`pwd`/dist"
-fi
-
-/scripts/apply-runtime-env.sh "`pwd`/dist"
-
-yarn start
@@ -6,5 +6,10 @@ WORKDIR /app

 COPY . .

+# Get the latest Git commit hash and set in package.json
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+  jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
+  mv tmp.json package.json
+
 RUN echo "Installing dependencies and building ajna-watcher-ts" && \
  yarn && yarn build
@@ -1,11 +1,20 @@
 FROM node:18.16.0-alpine3.16

-RUN apk --update --no-cache add git python3 alpine-sdk
+RUN apk --update --no-cache add git python3 alpine-sdk jq

 WORKDIR /app

 COPY . .

+# Get the latest Git commit hash and set it in package.json of all watchers
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+    find . -name 'package.json' -exec sh -c ' \
+    for packageFile; do \
+        jq --arg commitHash "$0" ".commitHash = \$commitHash" "$packageFile" > "$packageFile.tmp" && \
+        mv "$packageFile.tmp" "$packageFile"; \
+    done \
+    ' "$COMMIT_HASH" {} \;
+
 RUN echo "Building azimuth-watcher-ts" && \
    yarn && yarn build

@@ -6,5 +6,10 @@ WORKDIR /app

 COPY . .

+# Get the latest Git commit hash and set in package.json
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+  jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
+  mv tmp.json package.json
+
 RUN echo "Installing dependencies and building merkl-sushiswap-v3-watcher-ts" && \
  yarn && yarn build
@@ -6,5 +6,10 @@ WORKDIR /app

 COPY . .

+# Get the latest Git commit hash and set in package.json
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+  jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
+  mv tmp.json package.json
+
 RUN echo "Installing dependencies and building sushiswap-v3-watcher-ts" && \
  yarn && yarn build
@@ -32,7 +32,7 @@ RUN \

 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base
+    && apt-get -y install --no-install-recommends jq gettext-base git

 # [Optional] Uncomment if you want to install an additional version of node using nvm
 # ARG EXTRA_NODE_VERSION=10
@@ -2,7 +2,7 @@ version: "1.0"
 name: ajna
 description: "Ajna watcher stack"
 repos:
-  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.9
+  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.13
 containers:
  - cerc/watcher-ajna
 pods:
@@ -44,34 +44,42 @@ network:
     - 0.0.0.0:9000:9000
    azimuth-watcher-server:
     - 0.0.0.0:3001:3001
+     - 0.0.0.0:9001:9001
    censures-watcher-job-runner:
     - 0.0.0.0:9002:9002
    censures-watcher-server:
     - 0.0.0.0:3002:3002
+     - 0.0.0.0:9003:9003
    claims-watcher-job-runner:
     - 0.0.0.0:9004:9004
    claims-watcher-server:
     - 0.0.0.0:3003:3003
+     - 0.0.0.0:9005:9005
    conditional-star-release-watcher-job-runner:
     - 0.0.0.0:9006:9006
    conditional-star-release-watcher-server:
     - 0.0.0.0:3004:3004
+     - 0.0.0.0:9007:9007
    delegated-sending-watcher-job-runner:
     - 0.0.0.0:9008:9008
    delegated-sending-watcher-server:
     - 0.0.0.0:3005:3005
+     - 0.0.0.0:9009:9009
    ecliptic-watcher-job-runner:
     - 0.0.0.0:9010:9010
    ecliptic-watcher-server:
     - 0.0.0.0:3006:3006
+     - 0.0.0.0:9011:9011
    linear-star-release-watcher-job-runner:
     - 0.0.0.0:9012:9012
    linear-star-release-watcher-server:
     - 0.0.0.0:3007:3007
+     - 0.0.0.0:9013:9013
    polls-watcher-job-runner:
     - 0.0.0.0:9014:9014
    polls-watcher-server:
     - 0.0.0.0:3008:3008
+     - 0.0.0.0:9015:9015
    gateway-server:
     - 0.0.0.0:4000:4000
 ...
@@ -1,7 +1,7 @@
 version: "1.0"
 name: azimuth
 repos:
-  - github.com/cerc-io/azimuth-watcher-ts@v0.1.4
+  - github.com/cerc-io/azimuth-watcher-ts@0.1.6
 containers:
  - cerc/watcher-azimuth
 pods:
@@ -43,16 +43,18 @@ customized by editing the "spec" file generated by `laconic-so deploy init`.
 ```
 $ cat graph-node-spec.yml
 stack: graph-node
-ports:
-  graph-node:
-    - '8000:8000'
-    - '8001'
-    - '8020:8020'
-    - '8030'
-  ipfs:
-    - '8080'
-    - '4001'
-    - '5001:5001'
+network:
+  ports:
+    graph-node:
+      - '8000:8000'
+      - '8001'
+      - '8020:8020'
+      - '8030'
+      - '8040'
+    ipfs:
+      - '8080'
+      - '4001'
+      - '5001:5001'
 ...
 ```

@@ -64,7 +66,7 @@ laconic-so --stack graph-node deploy create --spec-file graph-node-spec.yml --de

 ## Start the stack

-Create an env file with the following values to be set before starting the stack:
+Update `config.env` file inside deployment directory with the following values before starting the stack:

 ```bash
 # Set ETH RPC endpoint the graph node will use
@@ -88,10 +90,13 @@ export GRAPH_ETHEREUM_REQUEST_RETRIES=
 # Maximum number of blocks to scan for triggers in each request (default: 2000)
 export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=

+# Maximum number of concurrent requests made against Ethereum for requesting transaction receipts during block ingestion (default: 1000)
+export GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS=
+
 # Ref: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
 ```

-Example env file:
+Example `config.env` file:

 ```bash
 export ETH_RPC_HOST=filecoin.chainup.net
@@ -104,12 +109,6 @@ export GRAPH_ETHEREUM_REQUEST_RETRIES=5
 export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=50
 ```

-Set the environment variables:
-
-```bash
-source <PATH_TO_ENV_FILE>
-```
-
 Deploy the stack:

 ```bash
@@ -3,7 +3,9 @@ name: graph-node
 description: "Stack for running graph-node"
 repos:
  - github.com/graphprotocol/graph-node
+  - github.com/cerc-io/watcher-ts@v0.2.92
 containers:
  - cerc/graph-node
+  - cerc/watcher-ts
 pods:
  - graph-node
@@ -14,7 +14,8 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.

 from stack_orchestrator.util import get_yaml
-from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
+from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand
+from stack_orchestrator.deploy.deployment_context import DeploymentContext
 from stack_orchestrator.deploy.stack_state import State
 from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command
 from stack_orchestrator.command_types import CommandOptions
@@ -2,7 +2,7 @@ version: "1.0"
 name: merkl-sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.11
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.14
 containers:
  - cerc/watcher-merkl-sushiswap-v3
 pods:
@@ -134,6 +134,29 @@ Note: Use `host.docker.internal` as host to access ports on the host machine

 Place the dashboard json files in grafana dashboards config directory (`monitoring-deployment/config/monitoring/grafana/dashboards`) in the deployment folder

+#### Graph Node Config
+
+For graph-node dashboard postgres datasource needs to be setup in `monitoring-deployment/config/monitoring/grafana/provisioning/datasources/graph-node-postgres.yml` (in deployment folder)
+
+```yml
+# graph-node-postgres.yml
+...
+datasources:
+  - name: Graph Node Postgres
+    type: postgres
+    jsonData:
+      # Set name to remote graph-node database name
+      database: graph-node
+      ...
+    # Set user to remote graph-node database username
+    user: graph-node
+    # Add URL for remote graph-node database
+    url: graph-node-db:5432
+    # Set password for graph-node database
+    secureJsonData:
+      password: 'password'
+```
+
 ### Env

 Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`):
@@ -156,6 +179,11 @@ Set the following env variables in the deployment env config file (`monitoring-d
  # Grafana server host URL (used in various links in alerts, etc.)
  # (Optional, default: http://localhost:3000)
  GF_SERVER_ROOT_URL=
+
+  
+  # RPC endpoint used by graph-node for upstream head metric
+  # (Optional, default: https://mainnet.infura.io/v3)
+  GRAPH_NODE_RPC_ENDPOINT=
  ```

 ## Start the stack
@@ -57,35 +57,35 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
    metrics_path: /metrics
    scheme: http
    static_configs:
-      - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_PORT']
+      - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_METRICS_PORT', 'AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'azimuth'
          chain: 'ethereum'
-      - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_PORT']
+      - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_METRICS_PORT', 'CENSURES_WATCHER_HOST:CENSURES_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'censures'
          chain: 'ethereum'
-      - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_PORT']
+      - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_METRICS_PORT', 'CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'claims'
          chain: 'ethereum'
-      - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_PORT']
+      - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_METRICS_PORT', 'CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'conditional_star_release'
          chain: 'ethereum'
-      - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_PORT']
+      - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_METRICS_PORT', 'DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'delegated_sending'
          chain: 'ethereum'
-      - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_PORT']
+      - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_METRICS_PORT', 'ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'ecliptic'
          chain: 'ethereum'
-      - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_PORT']
+      - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_METRICS_PORT', 'LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'linear_star_release'
          chain: 'ethereum'
-      - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_PORT']
+      - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_METRICS_PORT', 'POLLS_WATCHER_HOST:POLLS_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'polls'
          chain: 'ethereum'
@@ -95,11 +95,11 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
    metrics_path: /metrics
    scheme: http
    static_configs:
-      - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_PORT']
+      - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_METRICS_PORT', 'SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'sushiswap'
          chain: 'filecoin'
-      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_PORT']
+      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_METRICS_PORT', 'MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'merkl_sushiswap'
          chain: 'filecoin'
@@ -109,7 +109,7 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
    metrics_path: /metrics
    scheme: http
    static_configs:
-      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_PORT']
+      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_METRICS_PORT', 'AJNA_WATCHER_HOST:AJNA_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'ajna'
          chain: 'filecoin'
@@ -1,7 +1,7 @@
 version: "0.1"
 name: monitoring
 repos:
-  - github.com/cerc-io/watcher-ts@v0.2.81
+  - github.com/cerc-io/watcher-ts@v0.2.92
 containers:
  - cerc/watcher-ts
 pods:
@@ -1,10 +0,0 @@
-version: "1.0"
-name: snowballtools-base-backend
-description: "snowballtools-base-backend"
-repos:
-  - github.com/snowball-tools/snowballtools-base
-containers:
-  - cerc/webapp-base
-  - cerc/snowballtools-base-backend
-pods:
-  - snowballtools-base-backend
@@ -2,7 +2,7 @@ version: "1.0"
 name: sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.11
+  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.14
 containers:
  - cerc/watcher-sushiswap-v3
 pods:
@@ -29,16 +29,29 @@ def _image_needs_pushed(image: str):
    return image.endswith(":local")


+def _remote_tag_for_image(image: str, remote_repo_url: str):
+    # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
+    major_parts = image.split("/", 2)
+    image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
+    (image_name, image_version) = image_name_with_version.split(":")
+    if image_version == "local":
+        return f"{remote_repo_url}/{image_name}:deploy"
+    else:
+        return image
+
+
+# Note: do not add any calls this function
 def remote_image_exists(remote_repo_url: str, local_tag: str):
    docker = DockerClient()
    try:
-        remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
+        remote_tag = _remote_tag_for_image(local_tag, remote_repo_url)
        result = docker.manifest.inspect(remote_tag)
        return True if result else False
    except Exception:  # noqa: E722
        return False


+# Note: do not add any calls this function
 def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
    if not additional_tags:
        return
@@ -47,18 +60,20 @@ def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
        raise Exception(f"{local_tag} does not exist in {remote_repo_url}")

    docker = DockerClient()
-    remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
-    new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
+    remote_tag = _remote_tag_for_image(local_tag, remote_repo_url)
+    new_remote_tags = [_remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
    docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags)


-def remote_tag_for_image(image: str, remote_repo_url: str):
+def remote_tag_for_image_unique(image: str, remote_repo_url: str, deployment_id: str):
    # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
    major_parts = image.split("/", 2)
    image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
    (image_name, image_version) = image_name_with_version.split(":")
    if image_version == "local":
-        return f"{remote_repo_url}/{image_name}:deploy"
+        # Salt the tag with part of the deployment id to make it unique to this deployment
+        deployment_tag = deployment_id[-8:]
+        return f"{remote_repo_url}/{image_name}:deploy-{deployment_tag}"
    else:
        return image

@@ -73,14 +88,14 @@ def push_images_operation(command_context: DeployCommandContext, deployment_cont
    docker = DockerClient()
    for image in images:
        if _image_needs_pushed(image):
-            remote_tag = remote_tag_for_image(image, remote_repo_url)
+            remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id)
            if opts.o.verbose:
                print(f"Tagging {image} to {remote_tag}")
            docker.image.tag(image, remote_tag)
    # Run docker push commands to upload
    for image in images:
        if _image_needs_pushed(image):
-            remote_tag = remote_tag_for_image(image, remote_repo_url)
+            remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id)
            if opts.o.verbose:
                print(f"Pushing image {remote_tag}")
            docker.image.push(remote_tag)
@@ -26,7 +26,7 @@ from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variable
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment
 from stack_orchestrator.deploy.deploy_types import DeployEnvVars
 from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits
-from stack_orchestrator.deploy.images import remote_tag_for_image
+from stack_orchestrator.deploy.images import remote_tag_for_image_unique

 DEFAULT_VOLUME_RESOURCES = Resources({
    "reservations": {"storage": "2Gi"}
@@ -326,8 +326,11 @@ class ClusterInfo:
                if opts.o.debug:
                    print(f"Merged envs: {envs}")
                # Re-write the image tag for remote deployment
-                image_to_use = remote_tag_for_image(
-                    image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
+                # Note self.app_name has the same value as deployment_id
+                image_to_use = remote_tag_for_image_unique(
+                    image,
+                    self.spec.get_image_registry(),
+                    self.app_name) if self.spec.get_image_registry() is not None else image
                volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name)
                container = client.V1Container(
                    name=container_name,
@@ -22,16 +22,16 @@ echo "$(date +"%Y-%m-%d %T"): Stack started"
 # Verify that the fixturenet is up and running
 $TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd ps

+# Wait for the laconid endpoint to come up
+echo "Waiting for the RPC endpoint to come up"
+docker exec laconicd-laconicd-1 sh -c "curl --retry 20 --retry-delay 3 --retry-connrefused http://127.0.0.1:9473/api"
+
 # Get the fixturenet account address
 laconicd_account_address=$(docker exec laconicd-laconicd-1 laconicd keys list | awk '/- address:/ {print $3}')

 # Copy over config
 docker exec laconicd-cli-1 cp config.yml laconic-registry-cli/

-# Wait for the laconid endpoint to come up
-echo "Waiting for the RPC endpoint to come up"
-docker exec laconicd-laconicd-1 sh -c "curl --retry 20 --retry-delay 3 --retry-connrefused http://127.0.0.1:9473/api"
-
 # Run the tests
 echo "Running the tests"
 docker exec -e TEST_ACCOUNT=$laconicd_account_address laconicd-cli-1 sh -c 'cd laconic-registry-cli && yarn && yarn test'
Author	SHA1	Message	Date
telackey	3ba9436863	Add git to webapp base container.	2024-06-24 23:57:15 -05:00
dboreham	b4e82ebc19	Merge pull request 'Fix mainnet laconic deploy setup' (#850 ) from dboreham/fix-mainnet-laconic-init into main Reviewed-on: cerc-io/stack-orchestrator#850	2024-06-22 01:25:07 +00:00
dboreham	2364924a59	Fix mainnet laconic deploy setup	2024-06-21 19:24:33 -06:00
nabarun	a223797b4a	Update graph-node dashboard to show individual subgraph increase in query count (#846 ) Part of [Metrics and logging for GQL queries in watcher](https://www.notion.so/Metrics-and-logging-for-GQL-queries-in-watcher-928c692292b140a2a4f52cda9795df5e) Reviewed-on: cerc-io/stack-orchestrator#846 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>	2024-06-20 09:27:23 +00:00
nabarun	b8004e9870	Add Grafana panels for graph-node subgraph GQL queries (#845 ) Part of [Deploy v2 and updated v3 sushiswap subgraphs](https://www.notion.so/Deploy-v2-and-updated-v3-sushiswap-subgraphs-e331945fdeea487c890706fc22c6cc94) Reviewed-on: cerc-io/stack-orchestrator#845 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>	2024-06-19 10:40:54 +00:00
prathamesh	3fd99a1522	Handle race condition in laconic registry CLI tests (#843 ) Part of cerc-io/laconic-registry-cli#63 Reviewed-on: cerc-io/stack-orchestrator#843 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-06-19 06:41:58 +00:00
nabarun	842d999792	Add alert rules for secured secured-finance subgraph watcher (#842 ) Part of [Generate secured-finance subgraph watcher with codegen](https://www.notion.so/Generate-secured-finance-subgraph-watcher-with-codegen-2923413e0af54ea787c5435d6966f3bb) - Update watcher dashboard labels Reviewed-on: cerc-io/stack-orchestrator#842 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>	2024-06-18 12:28:02 +00:00
dboreham	d6a1fb3279	Merge pull request 'Fix image tag name' (#841 ) from dboreham/fix-remote-image-tags into main Reviewed-on: cerc-io/stack-orchestrator#841	2024-06-13 14:32:26 +00:00
dboreham	bf1eccd486	Fix image tag name	2024-06-13 08:31:45 -06:00
dboreham	3fb025b5c9	Make remote image tags unique to the deployment (#838 ) Reviewed-on: cerc-io/stack-orchestrator#838 Reviewed-by: Thomas E Lackey <telackey@noreply.git.vdb.to> Co-authored-by: David Boreham <david@bozemanpass.com> Co-committed-by: David Boreham <david@bozemanpass.com>	2024-06-13 03:26:58 +00:00
prathameshandnabarun	4acb06325b	Update watcher dashboard and config templates (#835 ) Part of [Metrics and logging for GQL queries in watcher](https://www.notion.so/Metrics-and-logging-for-GQL-queries-in-watcher-928c692292b140a2a4f52cda9795df5e) - Update watcher config templates after config refactoring - Mount watcher GQL query log files on volumes - Update watcher dashboard to - add a panel to show latest processed block number - use latest processed block from sync status for diff values Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Reviewed-on: cerc-io/stack-orchestrator#835 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-06-12 11:52:51 +00:00
dboreham	b80b647fa4	Merge pull request 'Remove files migrated to external repo' (#836 ) from dboreham/remove-snowball-stack into main Reviewed-on: cerc-io/stack-orchestrator#836	2024-06-07 03:17:48 +00:00
dboreham	9a1d3bb0f1	Remove files migrated to external repo	2024-06-06 20:44:51 -06:00
prathamesh	abc0c2423f	Add panels for GQL metrics to watcher dashboard (#834 ) Part of [Metrics and logging for GQL queries in watcher](https://www.notion.so/Metrics-and-logging-for-GQL-queries-in-watcher-928c692292b140a2a4f52cda9795df5e) Reviewed-on: cerc-io/stack-orchestrator#834 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-06-06 11:47:18 +00:00
nabarun	a322d6eed4	Add dashboard for graph-node subgraphs (#832 ) Part of [Deploy v2 and updated v3 sushiswap subgraphs](https://www.notion.so/Deploy-v2-and-updated-v3-sushiswap-subgraphs-e331945fdeea487c890706fc22c6cc94) - Add param `GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS` in graph-node stack - <https://github.com/graphprotocol/graph-node/blob/v0.31.0/docs/environment-variables.md#json-rpc-configuration-for-evm-chains> - Add dashboard for subgraphs deployment in graph-node - Show subgraph names in dashboard - Add watcher dashboard panel for showing watcher release version and commit hash Reviewed-on: cerc-io/stack-orchestrator#832 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>	2024-06-04 07:21:27 +00:00