Add alerts for graph-node subgraphs (#821 )

Part of [Deploy v2 and updated v3 sushiswap subgraphs](https://www.notion.so/Deploy-v2-and-updated-v3-sushiswap-subgraphs-e331945fdeea487c890706fc22c6cc94) Reviewed-on: #821 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
Merge pull request 'Remove legacy commands for docker startup' (#826 ) from dboreham/remove-old-docker-ci into main
2024-05-17 04:01:41 +00:00 · 2024-05-15 16:09:09 +00:00 · 2024-05-15 09:04:19 -06:00 · 2024-05-15 13:52:18 +00:00 · 2024-05-14 08:27:59 +00:00 · 2024-05-10 04:58:30 +00:00
43 changed files with 645 additions and 126 deletions
@@ -2,15 +2,13 @@ name: Fixturenet-Eth-Plugeth-Arm-Test

 on:
  push:
-    branches: 'main'
+    branches: '*'
    paths:
      - '!**'
      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-arm-test'
-  pull_request:
-    paths:
-      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-arm-test'
  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
-    - cron: '0 11 * * *'
+    - cron: '2 14 * * *'
+

 jobs:
  test:
@@ -2,15 +2,14 @@ name: Fixturenet-Eth-Plugeth-Test

 on:
  push:
-    branches: 'main'
-    paths:
-      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
-  pull_request:
+    branches: '*'
    paths:
+      - '!**'
      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
    - cron: '2 14 * * *'

+
 jobs:
  test:
    name: "Run an Ethereum plugeth fixturenet test"
@@ -2,13 +2,12 @@ name: Fixturenet-Eth-Test

 on:
  push:
-    branches: 'main'
-    paths:
-      - '.gitea/workflows/triggers/fixturenet-eth-test'
-  pull_request:
+    branches: '*'
    paths:
+      - '!**'
      - '.gitea/workflows/triggers/fixturenet-eth-test'

+
 jobs:
  test:
    name: "Run an Ethereum fixturenet test"
@@ -10,9 +10,6 @@ on:
    paths-ignore:
      - '.gitea/workflows/triggers/*'

-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock

 jobs:
  test:
@@ -41,10 +38,6 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
      - name: "Run deploy tests"
        run: ./tests/deploy/run-deploy-test.sh
      - name: Notify Vulcanize Slack on CI failure
@@ -10,10 +10,6 @@ on:
    paths-ignore:
      - '.gitea/workflows/triggers/*'

-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
 jobs:
  test:
    name: "Run webapp test suite"
@@ -43,10 +39,6 @@ jobs:
        run: ./scripts/build_shiv_package.sh
      - name: "Install wget" # 20240109 - Only needed until the executors are updated.
        run: apt update && apt install -y wget
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
      - name: "Run webapp tests"
        run: ./tests/webapp-test/run-webapp-test.sh
      - name: Notify Vulcanize Slack on CI failure
@@ -10,9 +10,6 @@ on:
    paths-ignore:
      - '.gitea/workflows/triggers/*'

-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock

 jobs:
  test:
@@ -41,10 +38,6 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
      - name: "Run smoke tests"
        run: ./tests/smoke-test/run-smoke-test.sh
      - name: Notify Vulcanize Slack on CI failure
@@ -1,3 +1,2 @@
 Change this file to trigger running the fixturenet-eth-plugeth-arm-test CI job
-1

@@ -1,4 +1,3 @@
 Change this file to trigger running the fixturenet-eth-plugeth-test CI job
 trigger
 trigger
-1
@@ -1,4 +1,2 @@
 Change this file to trigger running the fixturenet-eth-test CI job
-1
-1

@@ -6,10 +6,16 @@ services:
    restart: always
    environment:
      GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
+      CERC_GRAFANA_ALERTS_SUBGRAPH_IDS: ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
    volumes:
      - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
      - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards
+      - ../config/monitoring/update-grafana-alerts-config.sh:/update-grafana-alerts-config.sh
      - grafana_storage:/var/lib/grafana
+    user: root
+    entrypoint: ["bash", "-c"]
+    command: |
+      "/update-grafana-alerts-config.sh && /run.sh"
    ports:
      - "3000"
    healthcheck:
@@ -29,7 +29,7 @@ services:
    image: cerc/watcher-ajna:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-job-runner.sh"]
    volumes:
      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
    ports:
      - "9000"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -55,7 +55,7 @@ services:
    image: cerc/watcher-ajna:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-server.sh"]
    volumes:
      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -64,7 +64,7 @@ services:
      - "3008"
      - "9001"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -29,7 +29,7 @@ services:
    image: cerc/watcher-merkl-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-job-runner.sh"]
    volumes:
      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
    ports:
      - "9002:9000"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -55,7 +55,7 @@ services:
    image: cerc/watcher-merkl-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-server.sh"]
    volumes:
      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -64,7 +64,7 @@ services:
      - "127.0.0.1:3007:3008"
      - "9003:9001"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -29,7 +29,7 @@ services:
    image: cerc/watcher-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-job-runner.sh"]
    volumes:
      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
    ports:
      - "9000:9000"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -55,7 +55,7 @@ services:
    image: cerc/watcher-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-server.sh"]
    volumes:
      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -64,7 +64,7 @@ services:
      - "127.0.0.1:3008:3008"
      - "9001:9001"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -18,7 +18,7 @@
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 0,
-  "id": 4,
+  "id": 6,
  "links": [
    {
      "asDropdown": false,
@@ -55,7 +55,7 @@
        "x": 0,
        "y": 0
      },
-      "id": 1,
+      "id": 29,
      "panels": [
        {
          "datasource": {
@@ -103,7 +103,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@@ -173,7 +173,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@@ -242,7 +242,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@@ -438,7 +438,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@@ -518,7 +518,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@@ -629,7 +629,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@@ -730,7 +730,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@@ -854,7 +854,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@@ -1025,7 +1025,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@@ -1111,7 +1111,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@@ -1350,6 +1350,434 @@
          "title": "Num event-processing jobs",
          "type": "timeseries"
        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "Total number of failed ETH RPC requests by method and provider endpoint ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "fillOpacity": 90,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineWidth": 1,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 18
+          },
+          "id": 28,
+          "options": {
+            "barRadius": 0,
+            "barWidth": 0.97,
+            "fullHighlight": false,
+            "groupWidth": 1,
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "orientation": "auto",
+            "showValue": "auto",
+            "stacking": "none",
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            },
+            "xField": "Time",
+            "xTickLabelRotation": 0,
+            "xTickLabelSpacing": 0
+          },
+          "pluginVersion": "10.2.3",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "disableTextWrap": false,
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "watcher_eth_rpc_errors{job=~\"$job\", instance=~\"$watcher\"}",
+              "format": "time_series",
+              "fullMetaSearch": false,
+              "includeNullMetadata": true,
+              "instant": true,
+              "legendFormat": "{{method}}, {{provider}}",
+              "range": false,
+              "refId": "A",
+              "useBackend": false
+            }
+          ],
+          "title": "ETH RPC request failures",
+          "type": "barchart"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "Time taken by last ETH RPC requests by method and provider endpoint",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 18
+          },
+          "id": 26,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "watcher_eth_rpc_request_duration{job=~\"$job\", instance=~\"$watcher\"}",
+              "instant": false,
+              "legendFormat": "{{method}}, {{provider}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "ETH RPC request durations",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "Total number of failed ETH RPC requests by method and provider endpoint  (across all watchers)",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "fillOpacity": 90,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineWidth": 1,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 25
+          },
+          "id": 30,
+          "options": {
+            "barRadius": 0,
+            "barWidth": 0.97,
+            "fullHighlight": false,
+            "groupWidth": 1,
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "orientation": "auto",
+            "showValue": "auto",
+            "stacking": "none",
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            },
+            "xField": "Time",
+            "xTickLabelRotation": 0,
+            "xTickLabelSpacing": 0
+          },
+          "pluginVersion": "10.2.3",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "disableTextWrap": false,
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum by (method, provider) (watcher_eth_rpc_errors{chain=\"$target_chain\"})",
+              "format": "time_series",
+              "fullMetaSearch": false,
+              "includeNullMetadata": true,
+              "instant": true,
+              "legendFormat": "{{method}}, {{provider}}",
+              "range": false,
+              "refId": "A",
+              "useBackend": false
+            }
+          ],
+          "title": "Total ETH RPC request failures (across all watchers)",
+          "type": "barchart"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "Configured upstream ETH RPC endpoints",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "fixedColor": "blue",
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "align": "auto",
+                "cellOptions": {
+                  "type": "auto"
+                },
+                "filterable": false,
+                "inspect": false
+              },
+              "mappings": [
+                {
+                  "options": {
+                    "0": {
+                      "color": "red",
+                      "index": 0,
+                      "text": "inactive"
+                    },
+                    "1": {
+                      "color": "green",
+                      "index": 1,
+                      "text": "active"
+                    }
+                  },
+                  "type": "value"
+                }
+              ],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "blue",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "status"
+                },
+                "properties": [
+                  {
+                    "id": "custom.cellOptions",
+                    "value": {
+                      "type": "color-text"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 25
+          },
+          "id": 32,
+          "options": {
+            "cellHeight": "sm",
+            "footer": {
+              "countRows": false,
+              "enablePagination": false,
+              "fields": "",
+              "reducer": [
+                "sum"
+              ],
+              "show": false
+            },
+            "frameIndex": 0,
+            "showHeader": true,
+            "sortBy": [
+              {
+                "desc": true,
+                "displayName": "status"
+              }
+            ]
+          },
+          "pluginVersion": "10.2.3",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "watcher_config_upstream_endpoints{job=~\"$job\", instance=~\"$watcher\"}",
+              "format": "table",
+              "instant": true,
+              "legendFormat": "__auto",
+              "range": false,
+              "refId": "A"
+            }
+          ],
+          "title": "Configured upstream endpoints",
+          "transformations": [
+            {
+              "id": "labelsToFields",
+              "options": {
+                "mode": "columns"
+              }
+            },
+            {
+              "id": "organize",
+              "options": {
+                "excludeByName": {
+                  "Time": true,
+                  "__name__": true,
+                  "chain": true,
+                  "instance": true,
+                  "job": true
+                },
+                "includeByName": {},
+                "indexByName": {},
+                "renameByName": {
+                  "Value": "status"
+                }
+              }
+            }
+          ],
+          "type": "table"
+        },
        {
          "datasource": {
            "type": "prometheus",
@@ -1388,7 +1816,7 @@
            "h": 5,
            "w": 8,
            "x": 0,
-            "y": 18
+            "y": 32
          },
          "id": 18,
          "options": {
@@ -1410,7 +1838,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@@ -1533,7 +1961,7 @@
            "h": 5,
            "w": 8,
            "x": 8,
-            "y": 18
+            "y": 32
          },
          "id": 17,
          "options": {
@@ -1555,7 +1983,7 @@
            "textMode": "value_and_name",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@@ -1680,7 +2108,7 @@
            "h": 5,
            "w": 8,
            "x": 16,
-            "y": 18
+            "y": 32
          },
          "id": 19,
          "options": {
@@ -1702,7 +2130,7 @@
            "textMode": "auto",
            "wideLayout": true
          },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
          "targets": [
            {
              "datasource": {
@@ -1792,7 +2220,6 @@
            "uid": "PBFA97CFB590B2093"
          },
          "description": "Total number of subgraph entities loaded in event processing",
-          "hide": true,
          "fieldConfig": {
            "defaults": {
              "color": {
@@ -1847,8 +2274,9 @@
            "h": 6,
            "w": 8,
            "x": 0,
-            "y": 23
+            "y": 37
          },
+          "hide": true,
          "id": 8,
          "options": {
            "legend": {
@@ -1943,7 +2371,7 @@
            "h": 6,
            "w": 8,
            "x": 8,
-            "y": 23
+            "y": 37
          },
          "id": 9,
          "options": {
@@ -2039,7 +2467,7 @@
            "h": 6,
            "w": 8,
            "x": 16,
-            "y": 23
+            "y": 37
          },
          "id": 10,
          "options": {
@@ -2084,7 +2512,7 @@
    }
  ],
  "refresh": "10s",
-  "schemaVersion": 38,
+  "schemaVersion": 39,
  "tags": [
    "watcher"
  ],
@@ -2182,6 +2610,6 @@
  "timepicker": {},
  "timezone": "",
  "title": "Watchers",
-  "version": 1,
+  "version": 2,
  "weekStart": ""
 }
@@ -0,0 +1,64 @@
+apiVersion: 1
+groups:
+  - orgId: 1
+    name: subgraph
+    folder: SubgraphAlerts
+    interval: 30s
+    rules:
+      - uid: b2a9144b-6104-46fc-92b5-352f4e643c4c
+        title: subgraph_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                  type: prometheus
+                  uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: ethereum_chain_head_number - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                  - evaluator:
+                      params:
+                          - 15
+                          - 0
+                      type: gt
+                    operator:
+                      type: and
+                    query:
+                      params: []
+                    reducer:
+                      params: []
+                      type: avg
+                    type: query
+              datasource:
+                  name: Expression
+                  type: __expr__
+                  uid: __expr__
+              expression: diff
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: threshold
+        noDataState: OK
+        execErrState: Alerting
+        for: 5m
+        annotations:
+          summary: Subgraph deployment {{ index $labels "deployment" }} is falling behind head by {{ index $values "diff" }}
+        labels: {}
+        isPaused: false
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo Using CERC_GRAFANA_ALERTS_SUBGRAPH_IDS ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
+
+# Replace subgraph ids in subgraph alerting config
+# Note: Requires the grafana container to be run with user root
+sed -i "s/REPLACE_WITH_SUBGRAPH_IDS/$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS/g" /etc/grafana/provisioning/alerting/subgraph-alert-rules.yml
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -58,7 +58,7 @@

 [upstream]
  [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS

    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
    rpcClient = true
@@ -67,9 +67,9 @@
    isFEVM = true

    # Boolean flag to filter event logs by contracts
-    filterLogsByAddresses = true
+    filterLogsByAddresses = false
    # Boolean flag to filter event logs by topics
-    filterLogsByTopics = true
+    filterLogsByTopics = false

  [upstream.cache]
    name = "requests"
@@ -96,3 +96,6 @@
  # Max block range of historical processing after which it waits for completion of events processing
  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
  historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -58,7 +58,7 @@

 [upstream]
  [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS

    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
    rpcClient = true
@@ -67,7 +67,7 @@
    isFEVM = true

    # Boolean flag to filter event logs by contracts
-    filterLogsByAddresses = true
+    filterLogsByAddresses = false
    # Boolean flag to filter event logs by topics
    filterLogsByTopics = false

@@ -96,3 +96,6 @@
  # Max block range of historical processing after which it waits for completion of events processing
  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
  historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -58,7 +58,7 @@

 [upstream]
  [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS

    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
    rpcClient = true
@@ -67,9 +67,9 @@
    isFEVM = true

    # Boolean flag to filter event logs by contracts
-    filterLogsByAddresses = true
+    filterLogsByAddresses = false
    # Boolean flag to filter event logs by topics
-    filterLogsByTopics = true
+    filterLogsByTopics = false

  [upstream.cache]
    name = "requests"
@@ -96,3 +96,6 @@
  # Max block range of historical processing after which it waits for completion of events processing
  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
  historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3
@@ -9,7 +9,7 @@ STATUSES=(
  "beacon phase0"
  "beacon altair"
  "beacon bellatrix pre-merge"
-  "beacon post-merge"
+  "beacon bellatrix merge"
  "block number $MIN_BLOCK_NUM"
 )
 STATUS=0
@@ -69,13 +69,13 @@ while [ $STATUS -lt ${#STATUSES[@]} ]; do
      ;;
    2)
      result=`wget --no-check-certificate --quiet -O - "$LIGHTHOUSE_BASE_URL/eth/v2/beacon/blocks/head" | jq -r '.version'`
-      if [ ! -z "$result" ] && ([ "$result" == "altair" ] || [ "$result" == "bellatrix" ] || [ "$result" == "capella" ] || [ "$result" == "deneb" ]); then
+      if [ ! -z "$result" ] && ([ "$result" == "altair" ] || [ "$result" == "bellatrix" ]); then
        inc_status
      fi
      ;;
    3)
      result=`wget --no-check-certificate --quiet -O - "$LIGHTHOUSE_BASE_URL/eth/v2/beacon/blocks/head" | jq -r '.version'`
-      if [ ! -z "$result" ] && ([ "$result" == "bellatrix" ] || [ "$result" == "capella" ] || [ "$result" == "deneb" ]); then
+      if [ ! -z "$result" ] && [ "$result" == "bellatrix" ]; then
        inc_status
      fi
      ;;
@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v

 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
 ```

 ### Start the deployment
@@ -2,7 +2,7 @@ version: "1.0"
 name: ajna
 description: "Ajna watcher stack"
 repos:
-  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.1
+  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.6
 containers:
  - cerc/watcher-ajna
 pods:
@@ -1,9 +1,9 @@
 version: "1.0"
 name: erc20-watcher
 repos:
-  - git.vdb.to/cerc-io/go-ethereum@v1.13.14
+  - git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5
  - git.vdb.to/cerc-io/ipld-eth-db@v5
-  - git.vdb.to/cerc-io/ipld-eth-server@v5
+  - git.vdb.to/cerc-io/ipld-eth-server@v1.11.6-statediff-v5
  - github.com/dboreham/foundry
  - github.com/cerc-io/erc20-watcher-ts
 containers:
@@ -1,9 +1,9 @@
 version: "1.0"
 name: erc721-watcher
 repos:
-  - git.vdb.to/cerc-io/go-ethereum@v1.13.14
+  - git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5
  - git.vdb.to/cerc-io/ipld-eth-db@v5
-  - git.vdb.to/cerc-io/ipld-eth-server@v5
+  - git.vdb.to/cerc-io/ipld-eth-server@v1.11.6-statediff-v5
  - github.com/cerc-io/watcher-ts
 containers:
  - cerc/go-ethereum
@@ -2,9 +2,9 @@ version: "1.0"
 name: fixturenet-eth-loaded
 description: "Loaded Ethereum Fixturenet"
 repos:
-  - git.vdb.to/cerc-io/go-ethereum@v1.13.14
+  - git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5
  - git.vdb.to/cerc-io/tx-spammer
-  - git.vdb.to/cerc-io/ipld-eth-server@v5
+  - git.vdb.to/cerc-io/ipld-eth-server@v1.11.6-statediff-v5
  - git.vdb.to/cerc-io/ipld-eth-db@v5
  - git.vdb.to/cerc-io/lighthouse
 containers:
@@ -2,7 +2,7 @@ version: "1.2"
 name: fixturenet-eth-tx
 description: "Ethereum Fixturenet w/ tx-spammer"
 repos:
-  - git.vdb.to/cerc-io/go-ethereum@v1.13.14
+  - git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5
  - git.vdb.to/cerc-io/tx-spammer
  - git.vdb.to/cerc-io/lighthouse
  - github.com/dboreham/foundry
@@ -2,7 +2,7 @@ version: "1.1"
 name: fixturenet-eth
 description: "Ethereum Fixturenet"
 repos:
-  - git.vdb.to/cerc-io/go-ethereum@v1.13.14
+  - git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5
  - git.vdb.to/cerc-io/lighthouse
  - github.com/dboreham/foundry
 containers:
@@ -2,7 +2,7 @@ version: "1.0"
 name: fixturenet-optimism
 description: "Optimism Fixturenet"
 repos:
-  - git.vdb.to/cerc-io/go-ethereum@v1.13.14
+  - git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5
  - git.vdb.to/cerc-io/lighthouse
  - github.com/dboreham/foundry
  - github.com/ethereum-optimism/optimism@op-node/v1.3.0
@@ -3,8 +3,7 @@ name: fixturenet-payments
 description: "Stack to demonstrate payments between various services"
 repos:
  # fixturenet repos
-  - git.vdb.to/cerc-io/plugeth@statediff
-  - git.vdb.to/cerc-io/plugeth-statediff
+  - git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5
  - git.vdb.to/cerc-io/lighthouse
  - git.vdb.to/cerc-io/ipld-eth-db@v5
  # ipld-eth-server with payments -- to be used by ipld-eth-server-payments pod
@@ -25,12 +24,11 @@ repos:
  - github.com/cerc-io/erc20-watcher-ts
 containers:
  # fixturenet images
-  - cerc/plugeth-statediff
-  - cerc/plugeth
-  - cerc/fixturenet-eth-genesis
-  - cerc/fixturenet-plugeth-plugeth
+  - cerc/go-ethereum
  - cerc/lighthouse
  - cerc/lighthouse-cli
+  - cerc/fixturenet-eth-genesis
+  - cerc/fixturenet-eth-geth
  - cerc/fixturenet-eth-lighthouse
  - cerc/ipld-eth-db
  - cerc/ipld-eth-server
@@ -48,7 +46,7 @@ containers:
  - cerc/ponder
  - cerc/watcher-erc20
 pods:
-  - fixturenet-plugeth
+  - fixturenet-eth
  - ipld-eth-server-payments
  - ipld-eth-db
  - nitro-contracts
@@ -2,7 +2,7 @@ version: "1.0"
 name: fixturenet-pocket
 description: "A single node pocket chain that can serve relays from the geth-1 node in eth-fixturenet"
 repos:
-  - git.vdb.to/cerc-io/go-ethereum@v1.13.14
+  - git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5
  - git.vdb.to/cerc-io/lighthouse
  - github.com/pokt-network/pocket-core
  - github.com/pokt-network/pocket-core-deployments # contains the dockerfile
@@ -2,7 +2,7 @@ version: "1.2"
 name: mainnet-eth
 description: "Ethereum Mainnet"
 repos:
-  - git.vdb.to/cerc-io/go-ethereum@v1.13.14
+  - git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5
  - git.vdb.to/cerc-io/lighthouse
  - github.com/dboreham/foundry
  - git.vdb.to/cerc-io/keycloak-reg-api
@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v

 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
 ```

 ### Start the deployment
@@ -2,7 +2,7 @@ version: "1.0"
 name: merkl-sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.7
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.10
 containers:
  - cerc/watcher-merkl-sushiswap-v3
 pods:
@@ -1,7 +1,7 @@
 version: "1.0"
 name: mobymask-v2
 repos:
-  - git.vdb.to/cerc-io/go-ethereum@v1.13.14
+  - git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5
  - git.vdb.to/cerc-io/lighthouse
  - github.com/dboreham/foundry
  - github.com/ethereum-optimism/optimism@v1.0.4
@@ -113,21 +113,31 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
        labels:
          instance: 'ajna'
          chain: 'filecoin'
+
+  - job_name: graph-node
+    metrics_path: /metrics
+    scrape_interval: 30s
+    static_configs:
+      - targets: ['GRAPH_NODE_HOST:GRAPH_NODE_HOST_METRICS_PORT']
  ```

 Add scrape config as done above for any additional watcher to add it to the Watchers dashboard.

 ### Grafana alerts config

-Place the pre-configured watcher alerts rules in Grafana provisioning directory:
+Place the pre-configured alerts rules in Grafana provisioning directory:

  ```bash
+  # watcher alert rules
  cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
+
+  # subgraph alert rules
+  cp monitoring-watchers-deployment/config/monitoring/subgraph-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
  ```

 Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points

-Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/monitoring/grafana/provisioning/alerting/policies.yaml`) with appropriate object-matchers:
+Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/policies.yml`) with appropriate object-matchers:

  ```yml
  ...
@@ -135,7 +145,7 @@ Add corresponding routes to the notification policies config (`monitoring-watche
      - receiver: SlackNotifier
      object_matchers:
        # Add matchers below
-        - ['grafana_folder', '=', 'WatcherAlerts']
+        - ['grafana_folder', '=~', 'WatcherAlerts|SubgraphAlerts']
  ```

 ### Env
@@ -149,6 +159,9 @@ Set the following env variables in the deployment env config file (`monitoring-w
  # Grafana server host URL to be used
  # (Optional, default: http://localhost:3000)
  GF_SERVER_ROOT_URL=
+
+  # List of subgraph ids to configure alerts for (separated by |)
+  CERC_GRAFANA_ALERTS_SUBGRAPH_IDS=
  ```

 ## Start the stack
@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v

 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
 ```

 ### Start the deployment
@@ -2,7 +2,7 @@ version: "1.0"
 name: sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.7
+  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.10
 containers:
  - cerc/watcher-sushiswap-v3
 pods:
Author	SHA1	Message	Date
prathamesh	8f2da38183	Add alerts for graph-node subgraphs (#821 ) Lint Checks / Run linter (push) Successful in 45s Details Publish / Build and publish (push) Successful in 1m27s Details Webapp Test / Run webapp test suite (push) Successful in 3m50s Details Deploy Test / Run deploy test suite (push) Successful in 5m23s Details Smoke Test / Run basic test suite (push) Successful in 4m40s Details Database Test / Run database hosting test on kind/k8s (push) Successful in 12m11s Details Container Registry Test / Run contaier registry hosting test on kind/k8s (push) Successful in 4m34s Details External Stack Test / Run external stack test suite (push) Successful in 4m51s Details Fixturenet-Laconicd-Test / Run Laconicd fixturenet and Laconic CLI tests (push) Successful in 15m20s Details K8s Deploy Test / Run deploy test suite on kind/k8s (push) Successful in 11m10s Details Fixturenet-Eth-Plugeth-Arm-Test / Run an Ethereum plugeth fixturenet test (push) Failing after 3h12m58s Details Fixturenet-Eth-Plugeth-Test / Run an Ethereum plugeth fixturenet test (push) Failing after 3h13m0s Details Part of [Deploy v2 and updated v3 sushiswap subgraphs](https://www.notion.so/Deploy-v2-and-updated-v3-sushiswap-subgraphs-e331945fdeea487c890706fc22c6cc94) Reviewed-on: #821 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-05-17 04:01:41 +00:00
dboreham	254f95e59f	Merge pull request 'Remove legacy commands for docker startup' (#826 ) from dboreham/remove-old-docker-ci into main Lint Checks / Run linter (push) Successful in 50s Details Publish / Build and publish (push) Successful in 1m31s Details Webapp Test / Run webapp test suite (push) Successful in 4m12s Details Deploy Test / Run deploy test suite (push) Successful in 5m29s Details Smoke Test / Run basic test suite (push) Successful in 5m9s Details Fixturenet-Laconicd-Test / Run Laconicd fixturenet and Laconic CLI tests (push) Successful in 14m48s Details K8s Deploy Test / Run deploy test suite on kind/k8s (push) Successful in 10m13s Details Fixturenet-Eth-Plugeth-Arm-Test / Run an Ethereum plugeth fixturenet test (push) Failing after 3h12m49s Details Fixturenet-Eth-Plugeth-Test / Run an Ethereum plugeth fixturenet test (push) Failing after 3h12m50s Details Database Test / Run database hosting test on kind/k8s (push) Successful in 11m6s Details Container Registry Test / Run contaier registry hosting test on kind/k8s (push) Successful in 4m39s Details External Stack Test / Run external stack test suite (push) Successful in 4m54s Details Reviewed-on: #826	2024-05-15 16:09:09 +00:00
dboreham	0acb6ea6bc	Remove legacy commands for docker startup Lint Checks / Run linter (pull_request) Successful in 1m14s Details Deploy Test / Run deploy test suite (pull_request) Successful in 5m25s Details Webapp Test / Run webapp test suite (pull_request) Successful in 4m40s Details Smoke Test / Run basic test suite (pull_request) Successful in 4m55s Details K8s Deploy Test / Run deploy test suite on kind/k8s (pull_request) Successful in 10m8s Details	2024-05-15 09:04:19 -06:00
prathamesh	b9369a13e6	Update watcher dashboard with panels for ETH RPC request failures and durations (#825 ) Lint Checks / Run linter (push) Successful in 53s Details Publish / Build and publish (push) Successful in 1m33s Details Webapp Test / Run webapp test suite (push) Successful in 4m48s Details Smoke Test / Run basic test suite (push) Successful in 4m28s Details Deploy Test / Run deploy test suite (push) Successful in 5m36s Details K8s Deploy Test / Run deploy test suite on kind/k8s (push) Successful in 8m31s Details Fixturenet-Eth-Plugeth-Test / Run an Ethereum plugeth fixturenet test (push) Failing after 3h13m0s Details Fixturenet-Eth-Plugeth-Arm-Test / Run an Ethereum plugeth fixturenet test (push) Failing after 3h13m0s Details Part of [Regenerate ajna watcher with updated subgraph config ](https://www.notion.so/Regenerate-ajna-watcher-with-updated-subgraph-config-c9bbecb033024c13a7515c7f1efc3363) Requires [Add metrics to monitor errors and duration for ETH RPC requests](https://github.com/cerc-io/watcher-ts/pull/507) Reviewed-on: #825 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-05-15 13:52:18 +00:00
prathamesh	0b1eb8eb0f	Update ajna-watcher version in ajna stack (#824 ) Lint Checks / Run linter (push) Successful in 47s Details Publish / Build and publish (push) Successful in 1m37s Details Deploy Test / Run deploy test suite (push) Successful in 4m56s Details Webapp Test / Run webapp test suite (push) Successful in 5m4s Details Smoke Test / Run basic test suite (push) Successful in 4m46s Details K8s Deploy Test / Run deploy test suite on kind/k8s (push) Successful in 9m29s Details Fixturenet-Eth-Plugeth-Test / Run an Ethereum plugeth fixturenet test (push) Failing after 3h13m0s Details Fixturenet-Eth-Plugeth-Arm-Test / Run an Ethereum plugeth fixturenet test (push) Failing after 3h12m59s Details Database Test / Run database hosting test on kind/k8s (push) Successful in 10m11s Details Container Registry Test / Run contaier registry hosting test on kind/k8s (push) Successful in 4m11s Details External Stack Test / Run external stack test suite (push) Successful in 5m9s Details Fixturenet-Laconicd-Test / Run Laconicd fixturenet and Laconic CLI tests (push) Successful in 13m54s Details Part of [Regenerate ajna watcher with updated subgraph config](https://www.notion.so/Regenerate-ajna-watcher-with-updated-subgraph-config-c9bbecb033024c13a7515c7f1efc3363) Reviewed-on: #824 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-05-14 08:27:59 +00:00
nabarun	78092f5793	Update subgraph watcher stacks to configure multiple RPC endpoints (#822 ) Lint Checks / Run linter (push) Successful in 53s Details Publish / Build and publish (push) Successful in 1m30s Details Webapp Test / Run webapp test suite (push) Successful in 4m38s Details Smoke Test / Run basic test suite (push) Successful in 4m49s Details Deploy Test / Run deploy test suite (push) Successful in 5m55s Details Fixturenet-Laconicd-Test / Run Laconicd fixturenet and Laconic CLI tests (push) Successful in 13m54s Details K8s Deploy Test / Run deploy test suite on kind/k8s (push) Successful in 8m49s Details Fixturenet-Eth-Plugeth-Arm-Test / Run an Ethereum plugeth fixturenet test (push) Failing after 3h13m0s Details Fixturenet-Eth-Plugeth-Test / Run an Ethereum plugeth fixturenet test (push) Failing after 3h13m0s Details Database Test / Run database hosting test on kind/k8s (push) Successful in 10m43s Details Container Registry Test / Run contaier registry hosting test on kind/k8s (push) Successful in 4m40s Details External Stack Test / Run external stack test suite (push) Successful in 5m16s Details Part of [Ability to configure watchers with multiple RPC endpoints](https://www.notion.so/Ability-to-configure-watchers-with-multiple-RPC-endpoints-dc8d3ff4d647404ab718dfd5a4c9035c) Reviewed-on: #822 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>	2024-05-10 04:58:30 +00:00
prathameshandnabarun	247dbdd2f0	Update subgraph watcher versions for improved `eth_getLogs` calls (#820 ) Lint Checks / Run linter (push) Successful in 55s Details Publish / Build and publish (push) Successful in 1m36s Details Webapp Test / Run webapp test suite (push) Successful in 5m13s Details Deploy Test / Run deploy test suite (push) Successful in 5m32s Details Smoke Test / Run basic test suite (push) Successful in 5m3s Details Fixturenet-Laconicd-Test / Run Laconicd fixturenet and Laconic CLI tests (push) Successful in 15m22s Details K8s Deploy Test / Run deploy test suite on kind/k8s (push) Successful in 9m20s Details Fixturenet-Eth-Plugeth-Test / Run an Ethereum plugeth fixturenet test (push) Failing after 3h13m0s Details Fixturenet-Eth-Plugeth-Arm-Test / Run an Ethereum plugeth fixturenet test (push) Failing after 3h12m59s Details Database Test / Run database hosting test on kind/k8s (push) Successful in 10m50s Details Container Registry Test / Run contaier registry hosting test on kind/k8s (push) Successful in 4m15s Details External Stack Test / Run external stack test suite (push) Successful in 4m54s Details Part of [Investigate subgraph watchers lagging behind head](https://www.notion.so/Investigate-subgraph-watchers-lagging-behind-head-01b72294ca8e4f658e4c0e86b36d19e2) Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Reviewed-on: #820 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-05-07 04:18:45 +00:00