diff --git a/stack_orchestrator/data/compose/docker-compose-grafana.yml b/stack_orchestrator/data/compose/docker-compose-grafana.yml index 5054a0db..504c8d8d 100644 --- a/stack_orchestrator/data/compose/docker-compose-grafana.yml +++ b/stack_orchestrator/data/compose/docker-compose-grafana.yml @@ -18,6 +18,8 @@ services: "/update-grafana-alerts-config.sh && /run.sh" ports: - "3000" + extra_hosts: + - "host.docker.internal:host-gateway" healthcheck: test: ["CMD", "nc", "-vz", "localhost", "3000"] interval: 30s diff --git a/stack_orchestrator/data/compose/docker-compose-graph-node.yml b/stack_orchestrator/data/compose/docker-compose-graph-node.yml index e7f07e20..023fe03b 100644 --- a/stack_orchestrator/data/compose/docker-compose-graph-node.yml +++ b/stack_orchestrator/data/compose/docker-compose-graph-node.yml @@ -22,6 +22,7 @@ services: GRAPH_ETHEREUM_JSON_RPC_TIMEOUT: ${GRAPH_ETHEREUM_JSON_RPC_TIMEOUT:-180} GRAPH_ETHEREUM_REQUEST_RETRIES: ${GRAPH_ETHEREUM_REQUEST_RETRIES:-10} GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE: ${GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE:-2000} + GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS: ${GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS:-1000} entrypoint: ["bash", "-c"] # Wait for ETH RPC endpoint to be up when running with fixturenet-lotus command: | @@ -31,6 +32,7 @@ services: - "8001" - "8020" - "8030" + - "8040" healthcheck: test: ["CMD", "nc", "-vz", "localhost", "8020"] interval: 30s diff --git a/stack_orchestrator/data/compose/docker-compose-prom-server.yml b/stack_orchestrator/data/compose/docker-compose-prom-server.yml index 9095b6dc..594c48f0 100644 --- a/stack_orchestrator/data/compose/docker-compose-prom-server.yml +++ b/stack_orchestrator/data/compose/docker-compose-prom-server.yml @@ -28,15 +28,37 @@ services: extra_hosts: - "host.docker.internal:host-gateway" - chain-head-exporter: + ethereum-chain-head-exporter: image: cerc/watcher-ts:local restart: always working_dir: /app/packages/cli environment: - ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} - FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT} + ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT:-https://mainnet.infura.io/v3} ETH_RPC_API_KEY: ${CERC_INFURA_KEY} - PORT: ${CERC_METRICS_PORT} + command: ["sh", "-c", "yarn export-metrics:chain-heads"] + ports: + - '5000' + extra_hosts: + - "host.docker.internal:host-gateway" + + filecoin-chain-head-exporter: + image: cerc/watcher-ts:local + restart: always + working_dir: /app/packages/cli + environment: + ETH_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT:-https://api.node.glif.io/rpc/v1} + command: ["sh", "-c", "yarn export-metrics:chain-heads"] + ports: + - '5000' + extra_hosts: + - "host.docker.internal:host-gateway" + + graph-node-upstream-head-exporter: + image: cerc/watcher-ts:local + restart: always + working_dir: /app/packages/cli + environment: + ETH_RPC_ENDPOINT: ${GRAPH_NODE_RPC_ENDPOINT} command: ["sh", "-c", "yarn export-metrics:chain-heads"] ports: - '5000' diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/subgraphs-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/subgraphs-dashboard.json new file mode 100644 index 00000000..dc913e37 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/subgraphs-dashboard.json @@ -0,0 +1,1270 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Head block number for a deployment", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 62, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "deployment_head{deployment=~\"$subgraph_hash\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latest indexed block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Upstream head from graph-node ETH RPC endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\", instance=\"graph-node\", exported_chain=\"\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Upstream head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head from an external public endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 65, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\", instance=\"external\", exported_chain=\"\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "External head (filecoin)", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 7, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": true, + "current": false, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/eth_getLogs/", + "color": "#5794F2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "deployment_eth_rpc_request_duration{deployment=\"$subgraph_hash\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{method}}, {{provider}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "thresholds": [], + "timeRegions": [], + "title": "ETH RPC Request Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Block number of the most recent block synced from Ethereum", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 25, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "ethereum_chain_head_number", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Graph Node Head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "deployment_head{deployment=~\"$subgraph_hash\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\", instance=\"graph-node\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "deployment_head" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 66, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "deployment_head{deployment=~\"$subgraph_hash\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\", instance=\"external\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "deployment_head" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Block number of the latest block currently present in the chain head cache", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 63, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "chain_head_cache_latest_block", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Chain Cache Latest", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 2, + "text": "SUCCESS" + }, + "1": { + "color": "semi-dark-red", + "index": 1, + "text": "FAILED" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "orange", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 5 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "deployment_failed{deployment=~\"$subgraph_hash\",}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Deployment Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 6 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "deployment_head{deployment=~\"$subgraph_hash\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "ethereum_chain_head_number", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_node", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "ethereum_chain_head_number", + "operator": "-", + "right": "deployment_head" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 7, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/value/", + "color": "#FFB357" + }, + {} + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "deployment_block_processing_duration_sum{deployment=\"$subgraph_hash\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": " ", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Block Processing Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of successful ETH RPC requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "rate(endpoint_request{conn_type=~\"rpc\", result=~\"success\"}[$__rate_interval])", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{provider}}, {{req_type}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "ETH RPC successful requests rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of failed ETH RPC requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "rate(endpoint_request{conn_type=~\"rpc\", result=~\"failure\"}[$__rate_interval])", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{provider}}, {{req_type}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "ETH RPC failed requests rate", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "graph-node", + "subgraph" + ], + "templating": { + "list": [ + { + "current": { + "isNone": true, + "selected": false, + "value": "" + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PA8D093265C513DCC" + }, + "definition": "SELECT name\nFROM subgraphs.subgraph\nWHERE current_version IS NOT NULL;", + "description": "Name of deployed subgraph", + "hide": 0, + "includeAll": false, + "label": "Subgraph Name", + "multi": false, + "name": "subgraph_name", + "options": [], + "query": "SELECT name\nFROM subgraphs.subgraph\nWHERE current_version IS NOT NULL;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "PA8D093265C513DCC" + }, + "definition": "SELECT deployment\nFROM subgraphs.subgraph_version\n JOIN subgraphs.subgraph ON subgraph.current_version = subgraph_version.id\nWHERE subgraph.name = '$subgraph_name';", + "hide": 0, + "includeAll": false, + "label": "Subgraph Deployment", + "multi": false, + "name": "subgraph_hash", + "query": "SELECT deployment\nFROM subgraphs.subgraph_version\n JOIN subgraphs.subgraph ON subgraph.current_version = subgraph_version.id\nWHERE subgraph.name = '$subgraph_name';", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Graph Node Subgraphs", + "uid": "b54352dd-35f6-4151-97dc-265bab0c67e9", + "version": 18, + "weekStart": "" +} \ No newline at end of file diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json index 3a7771f4..89a9255f 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json @@ -57,6 +57,203 @@ }, "id": 29, "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "color-text" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 34, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "max(watcher_info{job=\"$job\", instance=\"$watcher\"}) by (version, commitHash)", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true + }, + "includeByName": {}, + "indexByName": { + "Time": 0, + "Value": 3, + "commitHash": 2, + "version": 1 + }, + "renameByName": { + "commitHash": "Commit hash", + "version": "Release" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + }, + { + "color": "red", + "value": 35 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "last_block_process_duration_seconds{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Last block process duration (s)", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -85,7 +282,7 @@ "h": 3, "w": 4, "x": 0, - "y": 1 + "y": 4 }, "id": 25, "options": { @@ -155,7 +352,7 @@ "h": 3, "w": 4, "x": 4, - "y": 1 + "y": 4 }, "id": 11, "options": { @@ -224,7 +421,7 @@ "h": 3, "w": 4, "x": 8, - "y": 1 + "y": 4 }, "id": 12, "options": { @@ -251,7 +448,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\"}", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\", instance=\"external\", exported_chain=\"\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -264,108 +461,6 @@ "title": "External head ($target_chain)", "type": "stat" }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "semi-dark-green", - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "red", - "value": 35 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "last_block_process_duration_seconds{job=~\"$job\", instance=~\"$watcher\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Last block process duration (s)", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -420,7 +515,7 @@ "h": 2.5, "w": 4, "x": 0, - "y": 4 + "y": 7 }, "id": 20, "options": { @@ -497,7 +592,7 @@ "h": 2, "w": 4, "x": 4, - "y": 4 + "y": 7 }, "id": 22, "options": { @@ -608,7 +703,7 @@ "h": 2, "w": 4, "x": 8, - "y": 4 + "y": 7 }, "id": 24, "options": { @@ -638,7 +733,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\"}", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\", instance=\"external\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -682,6 +777,107 @@ ], "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_block_num_events_total{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of events in the last block", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -711,7 +907,7 @@ "h": 3, "w": 8, "x": 4, - "y": 6 + "y": 9 }, "id": 13, "options": { @@ -836,7 +1032,7 @@ "h": 2.5, "w": 4, "x": 0, - "y": 6.5 + "y": 9.5 }, "id": 16, "options": { @@ -876,107 +1072,6 @@ "title": "Watcher sync mode", "type": "stat" }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 7 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_block_num_events_total{job=~\"$job\", instance=~\"$watcher\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Number of events in the last block", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -1006,7 +1101,7 @@ "h": 3, "w": 6, "x": 0, - "y": 9 + "y": 12 }, "id": 7, "options": { @@ -1092,7 +1187,7 @@ "h": 3, "w": 6, "x": 6, - "y": 9 + "y": 12 }, "id": 4, "options": { @@ -1256,9 +1351,9 @@ }, "gridPos": { "h": 6, - "w": 12, + "w": 8, "x": 0, - "y": 12 + "y": 15 }, "id": 33, "options": { @@ -1311,7 +1406,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\"}", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\", instance=\"external\"}", "hide": false, "instant": false, "legendFormat": "{{__name__}}", @@ -1432,9 +1527,9 @@ }, "gridPos": { "h": 6, - "w": 6, - "x": 12, - "y": 12 + "w": 8, + "x": 8, + "y": 15 }, "id": 5, "options": { @@ -1534,9 +1629,9 @@ }, "gridPos": { "h": 6, - "w": 6, - "x": 18, - "y": 12 + "w": 8, + "x": 16, + "y": 15 }, "id": 15, "options": { @@ -1631,10 +1726,10 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 0, - "y": 18 + "y": 21 }, "id": 28, "options": { @@ -1734,10 +1829,10 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 12, - "y": 18 + "y": 21 }, "id": 26, "options": { @@ -1826,10 +1921,10 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 0, - "y": 25 + "y": 27 }, "id": 30, "options": { @@ -1933,10 +2028,10 @@ ] }, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 12, - "y": 25 + "y": 27 }, "id": 32, "options": { @@ -2043,7 +2138,7 @@ "h": 5, "w": 8, "x": 0, - "y": 32 + "y": 33 }, "id": 18, "options": { @@ -2189,7 +2284,7 @@ "h": 5, "w": 8, "x": 8, - "y": 32 + "y": 33 }, "id": 17, "options": { @@ -2337,7 +2432,7 @@ "h": 5, "w": 8, "x": 16, - "y": 32 + "y": 33 }, "id": 19, "options": { @@ -2504,7 +2599,7 @@ "h": 6, "w": 8, "x": 0, - "y": 37 + "y": 38 }, "hide": true, "id": 8, @@ -2602,7 +2697,7 @@ "h": 6, "w": 8, "x": 8, - "y": 37 + "y": 38 }, "id": 9, "options": { @@ -2699,7 +2794,7 @@ "h": 6, "w": 8, "x": 16, - "y": 37 + "y": 38 }, "id": 10, "options": { diff --git a/stack_orchestrator/data/config/monitoring/grafana/provisioning/datasources/graph-node-postgres.yml b/stack_orchestrator/data/config/monitoring/grafana/provisioning/datasources/graph-node-postgres.yml new file mode 100644 index 00000000..d1604b06 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/provisioning/datasources/graph-node-postgres.yml @@ -0,0 +1,20 @@ +apiVersion: 1 + +datasources: + - name: Graph Node Postgres + type: postgres + jsonData: + database: graph-node + sslmode: 'disable' + maxOpenConns: 100 + maxIdleConns: 100 + maxIdleConnsAuto: true + connMaxLifetime: 14400 + postgresVersion: 1411 # 903=9.3, 1000=10, 1411=14.11 + timescaledb: false + user: graph-node + # # Add URL for graph-node database + # url: graph-node-db:5432 + # # Set password for graph-node database + # secureJsonData: + # password: 'password' diff --git a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml index 8229ff53..dea7052d 100644 --- a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml +++ b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml @@ -45,7 +45,18 @@ scrape_configs: metrics_path: /metrics scheme: http static_configs: - - targets: ['chain-head-exporter:5000'] + - targets: ['ethereum-chain-head-exporter:5000'] + labels: + instance: 'external' + chain: 'ethereum' + - targets: ['filecoin-chain-head-exporter:5000'] + labels: + instance: 'external' + chain: 'filecoin' + - targets: ['graph-node-upstream-head-exporter:5000'] + labels: + instance: 'graph-node' + chain: 'filecoin' - job_name: 'postgres' scrape_interval: 30s @@ -74,3 +85,11 @@ scrape_configs: # - targets: ['example-host:1317'] params: format: ['prometheus'] + + - job_name: graph-node + metrics_path: /metrics + scrape_interval: 30s + scheme: http + static_configs: + # Add graph-node targets to be monitored below + # - targets: ['graph-node:8040'] diff --git a/stack_orchestrator/data/config/monitoring/update-grafana-alerts-config.sh b/stack_orchestrator/data/config/monitoring/update-grafana-alerts-config.sh index b6ec932c..9f81203f 100755 --- a/stack_orchestrator/data/config/monitoring/update-grafana-alerts-config.sh +++ b/stack_orchestrator/data/config/monitoring/update-grafana-alerts-config.sh @@ -4,4 +4,6 @@ echo Using CERC_GRAFANA_ALERTS_SUBGRAPH_IDS ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS} # Replace subgraph ids in subgraph alerting config # Note: Requires the grafana container to be run with user root -sed -i "s/REPLACE_WITH_SUBGRAPH_IDS/$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS/g" /etc/grafana/provisioning/alerting/subgraph-alert-rules.yml +if [ -n "$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS" ]; then + sed -i "s/REPLACE_WITH_SUBGRAPH_IDS/$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS/g" /etc/grafana/provisioning/alerting/subgraph-alert-rules.yml +fi diff --git a/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml index c2025029..e5710565 100644 --- a/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml +++ b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml @@ -24,7 +24,7 @@ groups: uid: PBFA97CFB590B2093 disableTextWrap: false editorMode: code - expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"} + expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"} fullMetaSearch: false includeNullMetadata: true instant: true @@ -100,7 +100,7 @@ groups: uid: PBFA97CFB590B2093 disableTextWrap: false editorMode: code - expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"} + expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"} fullMetaSearch: false includeNullMetadata: true instant: true @@ -176,7 +176,7 @@ groups: uid: PBFA97CFB590B2093 disableTextWrap: false editorMode: code - expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"} + expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"} fullMetaSearch: false includeNullMetadata: true instant: true @@ -252,7 +252,7 @@ groups: uid: PBFA97CFB590B2093 disableTextWrap: false editorMode: code - expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"} + expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"} fullMetaSearch: false includeNullMetadata: true instant: true @@ -328,7 +328,7 @@ groups: uid: PBFA97CFB590B2093 disableTextWrap: false editorMode: code - expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"} + expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"} fullMetaSearch: false includeNullMetadata: true instant: true @@ -404,7 +404,7 @@ groups: uid: PBFA97CFB590B2093 disableTextWrap: false editorMode: code - expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"} + expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"} fullMetaSearch: false includeNullMetadata: true instant: true @@ -480,7 +480,7 @@ groups: uid: PBFA97CFB590B2093 disableTextWrap: false editorMode: code - expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"} + expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"} fullMetaSearch: false includeNullMetadata: true instant: true @@ -556,7 +556,7 @@ groups: uid: PBFA97CFB590B2093 disableTextWrap: false editorMode: code - expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"} + expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"} fullMetaSearch: false includeNullMetadata: true instant: true @@ -634,7 +634,7 @@ groups: uid: PBFA97CFB590B2093 disableTextWrap: false editorMode: code - expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"} + expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"} fullMetaSearch: false includeNullMetadata: true instant: true @@ -710,7 +710,7 @@ groups: uid: PBFA97CFB590B2093 disableTextWrap: false editorMode: code - expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"} + expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"} fullMetaSearch: false includeNullMetadata: true instant: true @@ -788,7 +788,7 @@ groups: uid: PBFA97CFB590B2093 disableTextWrap: false editorMode: code - expr: latest_block_number - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"} + expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"} fullMetaSearch: false includeNullMetadata: true instant: true diff --git a/stack_orchestrator/data/container-build/cerc-watcher-ajna/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-ajna/Dockerfile index 4369d50a..6b20eff6 100644 --- a/stack_orchestrator/data/container-build/cerc-watcher-ajna/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-watcher-ajna/Dockerfile @@ -6,5 +6,10 @@ WORKDIR /app COPY . . +# Get the latest Git commit hash and set in package.json +RUN COMMIT_HASH=$(git rev-parse HEAD) && \ + jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \ + mv tmp.json package.json + RUN echo "Installing dependencies and building ajna-watcher-ts" && \ yarn && yarn build diff --git a/stack_orchestrator/data/container-build/cerc-watcher-azimuth/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-azimuth/Dockerfile index eaf717aa..81253895 100644 --- a/stack_orchestrator/data/container-build/cerc-watcher-azimuth/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-watcher-azimuth/Dockerfile @@ -1,11 +1,20 @@ FROM node:18.16.0-alpine3.16 -RUN apk --update --no-cache add git python3 alpine-sdk +RUN apk --update --no-cache add git python3 alpine-sdk jq WORKDIR /app COPY . . +# Get the latest Git commit hash and set it in package.json of all watchers +RUN COMMIT_HASH=$(git rev-parse HEAD) && \ + find . -name 'package.json' -exec sh -c ' \ + for packageFile; do \ + jq --arg commitHash "$0" ".commitHash = \$commitHash" "$packageFile" > "$packageFile.tmp" && \ + mv "$packageFile.tmp" "$packageFile"; \ + done \ + ' "$COMMIT_HASH" {} \; + RUN echo "Building azimuth-watcher-ts" && \ yarn && yarn build diff --git a/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile index e09738ac..6ca9c7de 100644 --- a/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile @@ -6,5 +6,10 @@ WORKDIR /app COPY . . +# Get the latest Git commit hash and set in package.json +RUN COMMIT_HASH=$(git rev-parse HEAD) && \ + jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \ + mv tmp.json package.json + RUN echo "Installing dependencies and building merkl-sushiswap-v3-watcher-ts" && \ yarn && yarn build diff --git a/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/Dockerfile index ac6241c4..62e580a8 100644 --- a/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/Dockerfile @@ -6,5 +6,10 @@ WORKDIR /app COPY . . +# Get the latest Git commit hash and set in package.json +RUN COMMIT_HASH=$(git rev-parse HEAD) && \ + jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \ + mv tmp.json package.json + RUN echo "Installing dependencies and building sushiswap-v3-watcher-ts" && \ yarn && yarn build diff --git a/stack_orchestrator/data/stacks/ajna/stack.yml b/stack_orchestrator/data/stacks/ajna/stack.yml index 27d3229e..55f8b767 100644 --- a/stack_orchestrator/data/stacks/ajna/stack.yml +++ b/stack_orchestrator/data/stacks/ajna/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: ajna description: "Ajna watcher stack" repos: - - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.9 + - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.11 containers: - cerc/watcher-ajna pods: diff --git a/stack_orchestrator/data/stacks/azimuth/stack.yml b/stack_orchestrator/data/stacks/azimuth/stack.yml index 34e950c4..58ad3996 100644 --- a/stack_orchestrator/data/stacks/azimuth/stack.yml +++ b/stack_orchestrator/data/stacks/azimuth/stack.yml @@ -1,7 +1,7 @@ version: "1.0" name: azimuth repos: - - github.com/cerc-io/azimuth-watcher-ts@v0.1.4 + - github.com/cerc-io/azimuth-watcher-ts@0.1.5 containers: - cerc/watcher-azimuth pods: diff --git a/stack_orchestrator/data/stacks/graph-node/README.md b/stack_orchestrator/data/stacks/graph-node/README.md index 8c007367..47728ed8 100644 --- a/stack_orchestrator/data/stacks/graph-node/README.md +++ b/stack_orchestrator/data/stacks/graph-node/README.md @@ -43,16 +43,18 @@ customized by editing the "spec" file generated by `laconic-so deploy init`. ``` $ cat graph-node-spec.yml stack: graph-node -ports: - graph-node: - - '8000:8000' - - '8001' - - '8020:8020' - - '8030' - ipfs: - - '8080' - - '4001' - - '5001:5001' +network: + ports: + graph-node: + - '8000:8000' + - '8001' + - '8020:8020' + - '8030' + - '8040' + ipfs: + - '8080' + - '4001' + - '5001:5001' ... ``` @@ -64,7 +66,7 @@ laconic-so --stack graph-node deploy create --spec-file graph-node-spec.yml --de ## Start the stack -Create an env file with the following values to be set before starting the stack: +Update `config.env` file inside deployment directory with the following values before starting the stack: ```bash # Set ETH RPC endpoint the graph node will use @@ -88,10 +90,13 @@ export GRAPH_ETHEREUM_REQUEST_RETRIES= # Maximum number of blocks to scan for triggers in each request (default: 2000) export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE= +# Maximum number of concurrent requests made against Ethereum for requesting transaction receipts during block ingestion (default: 1000) +export GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS= + # Ref: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md ``` -Example env file: +Example `config.env` file: ```bash export ETH_RPC_HOST=filecoin.chainup.net @@ -104,12 +109,6 @@ export GRAPH_ETHEREUM_REQUEST_RETRIES=5 export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=50 ``` -Set the environment variables: - -```bash -source -``` - Deploy the stack: ```bash diff --git a/stack_orchestrator/data/stacks/graph-node/stack.yml b/stack_orchestrator/data/stacks/graph-node/stack.yml index ce45e965..262b7626 100644 --- a/stack_orchestrator/data/stacks/graph-node/stack.yml +++ b/stack_orchestrator/data/stacks/graph-node/stack.yml @@ -3,7 +3,9 @@ name: graph-node description: "Stack for running graph-node" repos: - github.com/graphprotocol/graph-node + - github.com/cerc-io/watcher-ts@v0.2.92 containers: - cerc/graph-node + - cerc/watcher-ts pods: - graph-node diff --git a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml index 8c692c3d..63483732 100644 --- a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: merkl-sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.11 + - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.12 containers: - cerc/watcher-merkl-sushiswap-v3 pods: diff --git a/stack_orchestrator/data/stacks/monitoring/README.md b/stack_orchestrator/data/stacks/monitoring/README.md index 99502902..f0d39be6 100644 --- a/stack_orchestrator/data/stacks/monitoring/README.md +++ b/stack_orchestrator/data/stacks/monitoring/README.md @@ -134,6 +134,29 @@ Note: Use `host.docker.internal` as host to access ports on the host machine Place the dashboard json files in grafana dashboards config directory (`monitoring-deployment/config/monitoring/grafana/dashboards`) in the deployment folder +#### Graph Node Config + +For graph-node dashboard postgres datasource needs to be setup in `monitoring-deployment/config/monitoring/grafana/provisioning/datasources/graph-node-postgres.yml` (in deployment folder) + +```yml +# graph-node-postgres.yml +... +datasources: + - name: Graph Node Postgres + type: postgres + jsonData: + # Set name to remote graph-node database name + database: graph-node + ... + # Set user to remote graph-node database username + user: graph-node + # Add URL for remote graph-node database + url: graph-node-db:5432 + # Set password for graph-node database + secureJsonData: + password: 'password' +``` + ### Env Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`): @@ -156,6 +179,11 @@ Set the following env variables in the deployment env config file (`monitoring-d # Grafana server host URL (used in various links in alerts, etc.) # (Optional, default: http://localhost:3000) GF_SERVER_ROOT_URL= + + + # RPC endpoint used by graph-node for upstream head metric + # (Optional, default: https://mainnet.infura.io/v3) + GRAPH_NODE_RPC_ENDPOINT= ``` ## Start the stack diff --git a/stack_orchestrator/data/stacks/monitoring/stack.yml b/stack_orchestrator/data/stacks/monitoring/stack.yml index 48605dc3..88a84b80 100644 --- a/stack_orchestrator/data/stacks/monitoring/stack.yml +++ b/stack_orchestrator/data/stacks/monitoring/stack.yml @@ -1,7 +1,7 @@ version: "0.1" name: monitoring repos: - - github.com/cerc-io/watcher-ts@v0.2.81 + - github.com/cerc-io/watcher-ts@v0.2.92 containers: - cerc/watcher-ts pods: diff --git a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml index 6f213df8..d43db222 100644 --- a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.11 + - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.12 containers: - cerc/watcher-sushiswap-v3 pods: