Add panels for ETH RPC request failures and durations to watcher dashboard

Update ajna-watcher version in ajna stack (#824 )
Part of [Regenerate ajna watcher with updated subgraph config](https://www.notion.so/Regenerate-ajna-watcher-with-updated-subgraph-config-c9bbecb033024c13a7515c7f1efc3363) Reviewed-on: cerc-io/stack-orchestrator#824 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
2024-05-15 12:34:15 +05:30 · 2024-05-14 08:27:59 +00:00 · 2024-05-10 04:58:30 +00:00
19 changed files with 279 additions and 53 deletions
--- a/stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml
+++ b/stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml
@ -29,7 +29,7 @@ services:
    image: cerc/watcher-ajna:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-job-runner.sh"]
    volumes:
      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@ -37,7 +37,7 @@ services:
    ports:
      - "9000"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
      interval: 20s
      timeout: 5s
      retries: 15
@ -55,7 +55,7 @@ services:
    image: cerc/watcher-ajna:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-server.sh"]
    volumes:
      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@ -64,7 +64,7 @@ services:
      - "3008"
      - "9001"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
      timeout: 5s
      retries: 15
--- a/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml
+++ b/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml
@ -29,7 +29,7 @@ services:
    image: cerc/watcher-merkl-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-job-runner.sh"]
    volumes:
      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@ -37,7 +37,7 @@ services:
    ports:
      - "9002:9000"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
      interval: 20s
      timeout: 5s
      retries: 15
@ -55,7 +55,7 @@ services:
    image: cerc/watcher-merkl-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-server.sh"]
    volumes:
      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@ -64,7 +64,7 @@ services:
      - "127.0.0.1:3007:3008"
      - "9003:9001"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
      timeout: 5s
      retries: 15
--- a/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml
+++ b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml
@ -29,7 +29,7 @@ services:
    image: cerc/watcher-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-job-runner.sh"]
    volumes:
      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@ -37,7 +37,7 @@ services:
    ports:
      - "9000:9000"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
      interval: 20s
      timeout: 5s
      retries: 15
@ -55,7 +55,7 @@ services:
    image: cerc/watcher-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-server.sh"]
    volumes:
      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@ -64,7 +64,7 @@ services:
      - "127.0.0.1:3008:3008"
      - "9001:9001"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
      timeout: 5s
      retries: 15
--- a/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json
+++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json
@ -18,7 +18,7 @@
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 0,
-  "id": 4,
+  "id": 3,
  "links": [
    {
      "asDropdown": false,
@ -55,7 +55,7 @@
        "x": 0,
        "y": 0
      },
-      "id": 1,
+      "id": 29,
      "panels": [
        {
          "datasource": {
@ -103,7 +103,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@ -173,7 +173,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@ -242,7 +242,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@ -438,7 +438,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@ -518,7 +518,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@ -629,7 +629,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@ -730,7 +730,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@ -854,7 +854,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@ -1025,7 +1025,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@ -1111,7 +1111,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@ -1410,7 +1410,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@ -1555,7 +1555,7 @@
            "textMode": "value_and_name",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@ -1702,7 +1702,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@ -1786,6 +1786,199 @@
          ],
          "type": "stat"
        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "Total number of failed ETH RPC requests by method and provider endpoint ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "fillOpacity": 90,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineWidth": 1,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 23
+          },
+          "id": 28,
+          "options": {
+            "barRadius": 0,
+            "barWidth": 0.97,
+            "fullHighlight": false,
+            "groupWidth": 1,
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "orientation": "auto",
+            "showValue": "auto",
+            "stacking": "none",
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            },
+            "xField": "Time",
+            "xTickLabelRotation": 0,
+            "xTickLabelSpacing": 0
+          },
+          "pluginVersion": "10.2.3",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "disableTextWrap": false,
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "watcher_eth_rpc_errors{job=~\"$job\", instance=~\"$watcher\"}",
+              "format": "time_series",
+              "fullMetaSearch": false,
+              "includeNullMetadata": true,
+              "instant": true,
+              "legendFormat": "{{method}}, {{provider}}",
+              "range": false,
+              "refId": "A",
+              "useBackend": false
+            }
+          ],
+          "title": "ETH RPC request failures",
+          "type": "barchart"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "Latest ETH RPC request durations by method and provider endpoint",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 23
+          },
+          "id": 26,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "watcher_eth_rpc_request_duration{job=~\"$job\", instance=~\"$watcher\"}",
+              "instant": false,
+              "legendFormat": "{{method}}, {{provider}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "ETH RPC request durations",
+          "type": "timeseries"
+        },
        {
          "datasource": {
            "type": "prometheus",
@ -1847,7 +2040,7 @@
            "h": 6,
            "w": 8,
            "x": 0,
-            "y": 23
+            "y": 30
          },
          "id": 8,
          "options": {
@ -1943,7 +2136,7 @@
            "h": 6,
            "w": 8,
            "x": 8,
-            "y": 23
+            "y": 30
          },
          "id": 9,
          "options": {
@ -2039,7 +2232,7 @@
            "h": 6,
            "w": 8,
            "x": 16,
-            "y": 23
+            "y": 30
          },
          "id": 10,
          "options": {
@ -2084,7 +2277,7 @@
    }
  ],
  "refresh": "10s",
-  "schemaVersion": 38,
+  "schemaVersion": 39,
  "tags": [
    "watcher"
  ],
@ -2182,6 +2375,6 @@
  "timepicker": {},
  "timezone": "",
  "title": "Watchers",
-  "version": 1,
+  "version": 2,
  "weekStart": ""
 }
--- a/stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh
+++ b/stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh
@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
--- a/stack_orchestrator/data/config/watcher-ajna/start-server.sh
+++ b/stack_orchestrator/data/config/watcher-ajna/start-server.sh
@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
--- a/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml
+++ b/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml
@ -58,7 +58,7 @@

 [upstream]
  [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS

    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
    rpcClient = true
@ -96,3 +96,6 @@
  # Max block range of historical processing after which it waits for completion of events processing
  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
  historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3
--- a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh
+++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh
@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
--- a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh
+++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh
@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
--- a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml
+++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml
@ -58,7 +58,7 @@

 [upstream]
  [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS

    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
    rpcClient = true
@ -96,3 +96,6 @@
  # Max block range of historical processing after which it waits for completion of events processing
  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
  historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3
--- a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh
+++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh
@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
--- a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh
+++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh
@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
--- a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml
+++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml
@ -58,7 +58,7 @@

 [upstream]
  [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS

    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
    rpcClient = true
@ -96,3 +96,6 @@
  # Max block range of historical processing after which it waits for completion of events processing
  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
  historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3
--- a/stack_orchestrator/data/stacks/ajna/README.md
+++ b/stack_orchestrator/data/stacks/ajna/README.md
@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v

 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
 ```

 ### Start the deployment
--- a/stack_orchestrator/data/stacks/ajna/stack.yml
+++ b/stack_orchestrator/data/stacks/ajna/stack.yml
@ -2,7 +2,7 @@ version: "1.0"
 name: ajna
 description: "Ajna watcher stack"
 repos:
-  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.3
+  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.5
 containers:
  - cerc/watcher-ajna
 pods:
--- a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md
+++ b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md
@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v

 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
 ```

 ### Start the deployment
--- a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml
+++ b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml
@ -2,7 +2,7 @@ version: "1.0"
 name: merkl-sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.9
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.10
 containers:
  - cerc/watcher-merkl-sushiswap-v3
 pods:
--- a/stack_orchestrator/data/stacks/sushiswap-v3/README.md
+++ b/stack_orchestrator/data/stacks/sushiswap-v3/README.md
@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v

 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
 ```

 ### Start the deployment
--- a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml
+++ b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml
@ -2,7 +2,7 @@ version: "1.0"
 name: sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.9
+  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.10
 containers:
  - cerc/watcher-sushiswap-v3
 pods: