From ef07280818a9fbefe1da1b53c54efa62d1613b80 Mon Sep 17 00:00:00 2001
From: Nabarun <nabarun@deepstacksoft.com>
Date: Wed, 29 May 2024 19:05:32 +0530
Subject: [PATCH] Add upstream block number panel and diff_upstream

---
 .../compose/docker-compose-graph-node.yml     |  11 +
 .../compose/docker-compose-prom-server.yml    |  18 +-
 .../dashboards/subgraphs-dashboard.json       | 448 ++++++++++++------
 .../grafana/dashboards/watcher-dashboard.json |   6 +-
 .../monitoring/prometheus/prometheus.yml      |  14 +-
 .../data/stacks/graph-node/README.md          |   4 +
 .../data/stacks/graph-node/stack.yml          |   3 +
 .../data/stacks/monitoring/stack.yml          |   3 +-
 8 files changed, 364 insertions(+), 143 deletions(-)

diff --git a/stack_orchestrator/data/compose/docker-compose-graph-node.yml b/stack_orchestrator/data/compose/docker-compose-graph-node.yml
index 023fe03b..dfcaa9bc 100644
--- a/stack_orchestrator/data/compose/docker-compose-graph-node.yml
+++ b/stack_orchestrator/data/compose/docker-compose-graph-node.yml
@@ -70,6 +70,17 @@ services:
       start_period: 3s
     ports:
       - "5432"
+  graph-node-chain-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${ETH_RPC_ENDPOINT:-http://lotus-node-1:1234/rpc/v1}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
 
 volumes:
   ipfs-import:
diff --git a/stack_orchestrator/data/compose/docker-compose-prom-server.yml b/stack_orchestrator/data/compose/docker-compose-prom-server.yml
index 9095b6dc..eb8ab971 100644
--- a/stack_orchestrator/data/compose/docker-compose-prom-server.yml
+++ b/stack_orchestrator/data/compose/docker-compose-prom-server.yml
@@ -28,15 +28,25 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
-  chain-head-exporter:
+  ethereum-chain-head-exporter:
     image: cerc/watcher-ts:local
     restart: always
     working_dir: /app/packages/cli
     environment:
-      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
-      FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT}
+      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT:-https://mainnet.infura.io/v3}
       ETH_RPC_API_KEY: ${CERC_INFURA_KEY}
-      PORT: ${CERC_METRICS_PORT}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  filecoin-chain-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT:-https://api.node.glif.io/rpc/v1}
     command: ["sh", "-c", "yarn export-metrics:chain-heads"]
     ports:
       - '5000'
diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/subgraphs-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/subgraphs-dashboard.json
index a268fe5f..be9b84c3 100644
--- a/stack_orchestrator/data/config/monitoring/grafana/dashboards/subgraphs-dashboard.json
+++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/subgraphs-dashboard.json
@@ -48,7 +48,7 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 4,
+        "h": 3,
         "w": 4,
         "x": 0,
         "y": 0
@@ -97,7 +97,7 @@
         "type": "prometheus",
         "uid": "PBFA97CFB590B2093"
       },
-      "description": "Block number of the most recent block synced from Ethereum",
+      "description": "Upstream head from graph-node ETH RPC endpoint",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -118,12 +118,12 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 2,
+        "h": 3,
         "w": 4,
         "x": 4,
         "y": 0
       },
-      "id": 25,
+      "id": 12,
       "options": {
         "colorMode": "background",
         "graphMode": "none",
@@ -148,18 +148,17 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "ethereum_chain_head_number",
+          "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\", instance=\"graph-node\"}",
           "fullMetaSearch": false,
-          "hide": false,
           "includeNullMetadata": true,
           "instant": false,
           "legendFormat": "__auto",
           "range": true,
-          "refId": "B",
+          "refId": "A",
           "useBackend": false
         }
       ],
-      "title": "Graph Node Head",
+      "title": "Upstream head",
       "type": "stat"
     },
     {
@@ -188,12 +187,12 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 4,
+        "h": 3,
         "w": 4,
         "x": 8,
         "y": 0
       },
-      "id": 12,
+      "id": 65,
       "options": {
         "colorMode": "background",
         "graphMode": "none",
@@ -218,7 +217,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\"}",
+          "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\", instance=\"external\"}",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "instant": false,
@@ -249,7 +248,7 @@
       "fill": 7,
       "fillGradient": 0,
       "gridPos": {
-        "h": 7,
+        "h": 8,
         "w": 12,
         "x": 12,
         "y": 0
@@ -332,6 +331,298 @@
         "align": false
       }
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "description": "Block number of the most recent block synced from Ethereum",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "blue",
+            "mode": "shades"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 4,
+        "x": 0,
+        "y": 3
+      },
+      "id": 25,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.2.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "ethereum_chain_head_number",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "B",
+          "useBackend": false
+        }
+      ],
+      "title": "Graph Node Head",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "blue",
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 8
+              },
+              {
+                "color": "red",
+                "value": 16
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 4,
+        "y": 3
+      },
+      "id": 24,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {
+          "valueSize": 25
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.2.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "code",
+          "expr": "deployment_head{deployment=~\"$subgraph_hash\"}",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\", instance=\"graph-node\"}",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "diff_upstream",
+      "transformations": [
+        {
+          "id": "calculateField",
+          "options": {
+            "alias": "",
+            "binary": {
+              "left": "latest_block_number",
+              "operator": "-",
+              "right": "deployment_head"
+            },
+            "mode": "binary",
+            "reduce": {
+              "reducer": "sum"
+            },
+            "replaceFields": true
+          }
+        }
+      ],
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "blue",
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 8
+              },
+              {
+                "color": "red",
+                "value": 16
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 8,
+        "y": 3
+      },
+      "id": 66,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {
+          "valueSize": 25
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.2.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "code",
+          "expr": "deployment_head{deployment=~\"$subgraph_hash\"}",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\", instance=\"external\"}",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "diff_external",
+      "transformations": [
+        {
+          "id": "calculateField",
+          "options": {
+            "alias": "",
+            "binary": {
+              "left": "latest_block_number",
+              "operator": "-",
+              "right": "deployment_head"
+            },
+            "mode": "binary",
+            "reduce": {
+              "reducer": "sum"
+            },
+            "replaceFields": true
+          }
+        }
+      ],
+      "type": "stat"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -358,10 +649,10 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 2,
+        "h": 3,
         "w": 4,
         "x": 4,
-        "y": 2
+        "y": 5
       },
       "id": 63,
       "options": {
@@ -455,8 +746,8 @@
       "gridPos": {
         "h": 3,
         "w": 4,
-        "x": 0,
-        "y": 4
+        "x": 8,
+        "y": 5
       },
       "id": 20,
       "options": {
@@ -530,10 +821,10 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 3,
+        "h": 2,
         "w": 4,
-        "x": 4,
-        "y": 4
+        "x": 0,
+        "y": 6
       },
       "id": 22,
       "options": {
@@ -586,7 +877,7 @@
           "refId": "B"
         }
       ],
-      "title": "Diff node head",
+      "title": "diff_node_head",
       "transformations": [
         {
           "id": "calculateField",
@@ -607,117 +898,6 @@
       ],
       "type": "stat"
     },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "blue",
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "orange",
-                "value": 8
-              },
-              {
-                "color": "red",
-                "value": 16
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 8,
-        "y": 4
-      },
-      "id": 24,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "text": {
-          "valueSize": 25
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.3",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "deployment_head{deployment=~\"$subgraph_hash\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "code",
-          "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "Diff external",
-      "transformations": [
-        {
-          "id": "calculateField",
-          "options": {
-            "alias": "",
-            "binary": {
-              "left": "latest_block_number",
-              "operator": "-",
-              "right": "deployment_head"
-            },
-            "mode": "binary",
-            "reduce": {
-              "reducer": "sum"
-            },
-            "replaceFields": true
-          }
-        }
-      ],
-      "type": "stat"
-    },
     {
       "aliasColors": {},
       "bars": false,
@@ -739,7 +919,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 7
+        "y": 8
       },
       "hiddenSeries": false,
       "id": 34,
@@ -885,7 +1065,7 @@
         "h": 7,
         "w": 12,
         "x": 0,
-        "y": 14
+        "y": 15
       },
       "id": 28,
       "options": {
@@ -984,7 +1164,7 @@
         "h": 7,
         "w": 12,
         "x": 12,
-        "y": 14
+        "y": 15
       },
       "id": 64,
       "options": {
diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json
index 3a7771f4..544dfa07 100644
--- a/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json
+++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json
@@ -251,7 +251,7 @@
               },
               "disableTextWrap": false,
               "editorMode": "builder",
-              "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\"}",
+              "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\", instance=\"external\"}",
               "fullMetaSearch": false,
               "includeNullMetadata": true,
               "instant": false,
@@ -638,7 +638,7 @@
               },
               "disableTextWrap": false,
               "editorMode": "builder",
-              "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\"}",
+              "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\", instance=\"external\"}",
               "fullMetaSearch": false,
               "includeNullMetadata": true,
               "instant": false,
@@ -1311,7 +1311,7 @@
                 "uid": "PBFA97CFB590B2093"
               },
               "editorMode": "code",
-              "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\"}",
+              "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\", instance=\"external\"}",
               "hide": false,
               "instant": false,
               "legendFormat": "{{__name__}}",
diff --git a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml
index 281b6e58..6fb538b3 100644
--- a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml
+++ b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml
@@ -45,7 +45,19 @@ scrape_configs:
     metrics_path: /metrics
     scheme: http
     static_configs:
-      - targets: ['chain-head-exporter:5000']
+      - targets: ['ethereum-chain-head-exporter:5000']
+        labels:
+          instance: 'external'
+          chain: 'ethereum'
+      - targets: ['filecoin-chain-head-exporter:5000']
+        labels:
+          instance: 'external'
+          chain: 'filecoin'
+      # # Add graph-node chain-head-exported targets to be monitored below
+      # - targets: ['graph-node-chain-head-exporter:5000']
+      #   labels:
+      #     instance: 'graph-node'
+      #     chain: 'filecoin'
 
   - job_name: 'postgres'
     scrape_interval: 30s
diff --git a/stack_orchestrator/data/stacks/graph-node/README.md b/stack_orchestrator/data/stacks/graph-node/README.md
index 47728ed8..5b24c5c8 100644
--- a/stack_orchestrator/data/stacks/graph-node/README.md
+++ b/stack_orchestrator/data/stacks/graph-node/README.md
@@ -75,6 +75,9 @@ Update `config.env` file inside deployment directory with the following values b
 export ETH_RPC_HOST=
 export ETH_RPC_PORT=
 
+# RPC endpoint used by graph-node for exposing upstream head as metrics
+export ETH_RPC_ENDPOINT=
+
 # The etherum network(s) graph-node will connect to
 # Set this to a space-separated list of the networks where each entry has the form NAME:URL
 export ETH_NETWORKS=
@@ -101,6 +104,7 @@ Example `config.env` file:
 ```bash
 export ETH_RPC_HOST=filecoin.chainup.net
 export ETH_RPC_PORT=443
+export ETH_RPC_ENDPOINT=https://filecoin.chainup.net
 
 export ETH_NETWORKS=filecoin:https://filecoin.chainup.net/rpc/v1
 
diff --git a/stack_orchestrator/data/stacks/graph-node/stack.yml b/stack_orchestrator/data/stacks/graph-node/stack.yml
index ce45e965..f85298d2 100644
--- a/stack_orchestrator/data/stacks/graph-node/stack.yml
+++ b/stack_orchestrator/data/stacks/graph-node/stack.yml
@@ -3,7 +3,10 @@ name: graph-node
 description: "Stack for running graph-node"
 repos:
   - github.com/graphprotocol/graph-node
+  # TODO: Set latest version
+  - github.com/cerc-io/watcher-ts
 containers:
   - cerc/graph-node
+  - cerc/watcher-ts
 pods:
   - graph-node
diff --git a/stack_orchestrator/data/stacks/monitoring/stack.yml b/stack_orchestrator/data/stacks/monitoring/stack.yml
index 48605dc3..2e9426af 100644
--- a/stack_orchestrator/data/stacks/monitoring/stack.yml
+++ b/stack_orchestrator/data/stacks/monitoring/stack.yml
@@ -1,7 +1,8 @@
 version: "0.1"
 name: monitoring
 repos:
-  - github.com/cerc-io/watcher-ts@v0.2.81
+  # Set latest version
+  - github.com/cerc-io/watcher-ts
 containers:
   - cerc/watcher-ts
 pods: