cerc-io/stack-orchestrator
17
1
Fork 3
Code Issues 113 Pull Requests 18 Actions Packages Projects Releases 480 Wiki Activity

Use external head for subgraph alerts and show blockProcessingOffset on watcher dashboard #861

Open
prathamesh wants to merge 4 commits from deep-stack/stack-orchestrator:pm-update-monitoring into main
pull from: deep-stack/stack-orchestrator:pm-update-monitoring
merge into: cerc-io:main
Conversation 0 Commits 4 Files Changed 5 +179 -79
5 changed files with 179 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
stack_orchestrator/data/compose/docker-compose-graph-node.yml
View File

@ -1,5 +1,6 @@
services:
graph-node:
restart: unless-stopped
image: cerc/graph-node:local
depends_on:
db:
@ -40,6 +41,7 @@ services:
retries: 10
start_period: 3s
ipfs:
restart: unless-stopped
image: ipfs/kubo:master-2023-02-20-714a968
volumes:
- ipfs-import:/import
@ -49,6 +51,7 @@ services:
- "4001"
- "5001"
db:
restart: unless-stopped
image: postgres:14-alpine
volumes:
- db-data:/var/lib/postgresql/data

15
stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json
View File

@ -2866,6 +2866,19 @@
"legendFormat": "{{field}}",
"range": true,
"refId": "E"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"jobqueue\", field=\"block_processing_offset\"}",
"hide": false,
"instant": false,
"legendFormat": "{{field}}",
"range": true,
"refId": "F"
}
],
"title": "Watcher config (job queue)",
@ -3284,6 +3297,6 @@
"timepicker": {},
"timezone": "",
"title": "Watchers",
"version": 5,
"version": 6,
"weekStart": ""
}

4
stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml
View File

@ -49,6 +49,10 @@ scrape_configs:
labels:
instance: 'external'
chain: 'ethereum'
- targets: ['sepolia-chain-head-exporter:5000']
labels:
instance: 'external'
chain: 'sepolia'
- targets: ['filecoin-chain-head-exporter:5000']
labels:
instance: 'external'

2
stack_orchestrator/data/config/monitoring/subgraph-alert-rules.yml
View File

@ -19,7 +19,7 @@ groups:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: ethereum_chain_head_number - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
expr: label_replace(latest_block_number{instance="external"}, "network", "$1", "chain", "(.*)") - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
instant: true
intervalMs: 1000
legendFormat: __auto

234
stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml
View File

@ -695,82 +695,6 @@ groups:
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: merkl_sushiswap_diff_external
title: merkl_sushiswap_watcher_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
useBackend: false
- refId: latest_external
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: latest_block_number{chain="filecoin"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_external
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
# Ajna
- uid: ajna_diff_external
@ -849,7 +773,7 @@ groups:
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
# Secured Finance
- uid: secured_finance_diff_external
title: secured_finance_watcher_head_tracking
@ -927,3 +851,159 @@ groups:
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
# Borg
- uid: borg_diff_external
title: borg_watcher_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="borg", instance="borg", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
useBackend: false
- refId: latest_external
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: latest_block_number{chain="sepolia"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_external
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
# Steer Protocol
- uid: steer_protocol_diff_external
title: steer_protocol_watcher_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="steer-protocol", instance="steer-protocol", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
useBackend: false
- refId: latest_external
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: latest_block_number{chain="filecoin"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_external
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false