Compare commits

..
Author SHA1 Message Date
nabarun db8aec52aa Pin commit hash of asset list repo in osmosis frontend app (#703) 2024-01-15 16:56:06 +05:30
telackey b83030f63b Use custom act with gitea. (#700)
* Use custom act with gitea.

* Make sure wget is available

* Fix repo url
2024-01-09 22:53:43 -06:00
prathamesh0 a3eb3c0bb0 Setup basic alerting for watchers in monitoring stack (#698)
* Provision Grafana alert contactpoints and policies for Slack

* Add watcher alert rules

* Update watcher monitoring instructions

* Add listening port flag to node exporter command

* Add reference links
2024-01-08 17:25:30 +05:30
nabarun eae2af7ccc Upgrade watcher versions (#699) 2024-01-08 11:51:54 +05:30
telackey 837e443800 Support application removal requests. (#697)
* Support application removal request.

* Git should never prompt when deploying a webapp
2023-12-21 18:05:40 -06:00
prathamesh0 a57b0cdd26 Add a stack for prom node exporter and its dashboard in monitoring stack (#696)
* Add a stack for Prometheus node exporter

* Add node exporter dashboard to monitoring stack
2023-12-21 15:15:03 +05:30
22 changed files with 25157 additions and 38 deletions
+2
View File
@@ -41,6 +41,8 @@ jobs:
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
run: ./scripts/build_shiv_package.sh
- name: "Install wget" # 20240109 - Only needed until the executors are updated.
run: apt update && apt install -y wget
- name: Start dockerd # Also needed until we can incorporate into the executor
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
@@ -0,0 +1,15 @@
version: '3.8'
services:
node-exporter:
image: prom/node-exporter:latest
restart: unless-stopped
command:
- '--web.listen-address=:9100'
- '--path.rootfs=/host'
- '--collector.systemd'
- '--collector.processes'
network_mode: host
pid: host
volumes:
- '/:/host:ro,rslave'
@@ -6,6 +6,7 @@ services:
restart: on-failure
environment:
- NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL}
- ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438
- NEXT_PUBLIC_BASEPATH=/apps/osmosis
- NEXT_PUBLIC_URBIT_DEPLOYMENT=true
working_dir: /app/packages/web
@@ -6,6 +6,7 @@ services:
restart: on-failure
environment:
- NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL}
- ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438
working_dir: /app/packages/web
command: ["./build-app.sh"]
volumes:
@@ -0,0 +1,14 @@
# https://www.clever-cloud.com/blog/features/2021/12/03/slack-alerts-for-grafana/
apiVersion: 1
contactPoints:
- orgId: 1
name: SlackNotifier
receivers:
- uid: a71b06e3-58b6-41fe-af65-fbbb29653951
type: slack
settings:
# Slack hook URL (see https://api.slack.com/messaging/webhooks)
url: <YOUR_SLACK_HOOK_URL>
disableResolveMessage: false
@@ -0,0 +1,15 @@
# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-notification-policy/
apiVersion: 1
policies:
- orgId: 1
receiver: grafana-default-email
group_by:
- grafana_folder
- alertname
routes:
- receiver: SlackNotifier
object_matchers:
# Add matchers below
# - ['grafana_folder', '=', 'MyAlerts']
@@ -11,6 +11,13 @@ scrape_configs:
static_configs:
- targets: ['localhost:9090']
- job_name: node
static_configs:
# Add node-exporter targets to be monitored below
# - targets: ['example-host:9100']
# labels:
# instance: 'my-host'
- job_name: 'blackbox'
scrape_interval: 10s
metrics_path: /probe
@@ -0,0 +1,973 @@
# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-grafana-managed-rule/
apiVersion: 1
groups:
- orgId: 1
name: watcher
folder: WatcherAlerts
interval: 30s
rules:
# Azimuth
- uid: azimuth_diff_external
title: azimuth_watcher_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
useBackend: false
- refId: latest_external
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: latest_block_number{chain="ethereum"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: censures_diff_external
title: censures_watcher_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
useBackend: false
- refId: latest_external
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: latest_block_number{chain="ethereum"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: claims_diff_external
title: claims_watcher_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
useBackend: false
- refId: latest_external
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: latest_block_number{chain="ethereum"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: conditional_star_release_diff_external
title: conditional_star_release_watcher_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
useBackend: false
- refId: latest_external
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: latest_block_number{chain="ethereum"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: delegated_sending_diff_external
title: delegated_sending_watcher_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
useBackend: false
- refId: latest_external
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: latest_block_number{chain="ethereum"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: ecliptic_diff_external
title: ecliptic_watcher_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
useBackend: false
- refId: latest_external
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: latest_block_number{chain="ethereum"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: linear_star_release_diff_external
title: linear_star_release_watcher_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
useBackend: false
- refId: latest_external
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: latest_block_number{chain="ethereum"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: polls_diff_external
title: polls_watcher_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
useBackend: false
- refId: latest_external
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: latest_block_number{chain="ethereum"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
# Sushi
- uid: sushiswap_diff_external
title: sushiswap_watcher_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
useBackend: false
- refId: latest_external
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: latest_block_number{chain="filecoin"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: merkl_sushiswap_diff_external
title: merkl_sushiswap_watcher_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
useBackend: false
- refId: latest_external
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: latest_block_number{chain="filecoin"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
panelId: 24
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
__dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
__panelId__: "24"
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
@@ -2,4 +2,6 @@
# Build a local version of the task executor for act-runner
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/act-runner/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR}
cd ${CERC_REPO_BASE_DIR}/hosting/act-runner
docker build -t cerc/act-runner-task-executor:local -f Dockerfile.task-executor ${build_command_args} .
@@ -2,7 +2,7 @@ version: "1.0"
name: merkl-sushiswap-v3
description: "SushiSwap v3 watcher stack"
repos:
- github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.4
- github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.6
containers:
- cerc/watcher-merkl-sushiswap-v3
pods:
@@ -2,9 +2,12 @@
* Instructions to setup and run a Prometheus server and Grafana dashboard
* Comes with the following built-in exporters / dashboards:
* [Prometheus Blackbox Exporter](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints
* [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics
* Chain Head Exporter - for tracking chain heads given external ETH RPC endpoints
* Watchers dashboard
* [Prometheus Blackbox](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints
* [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics
* [PostgreSQL Database](https://grafana.com/grafana/dashboards/9628-postgresql-database/) - for monitoring Postgres dbs
* [Node Exporter Full](https://grafana.com/grafana/dashboards/1860-node-exporter-full/) - for monitoring system metrics
* See [monitoring-watchers.md](./monitoring-watchers.md) for an example usage of the stack with pre-configured dashboards for watchers
## Setup
@@ -72,7 +75,20 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de
- targets: ['<METRICS_ENDPOINT_HOST>:<METRICS_ENDPOINT_PORT>']
```
* Blackbox: update the `blackbox` job to add any endpoints to be monitored on the Blackbox dashboard:
* Node exporter: update the `node` job to add any node-exporter targets to be monitored:
```yml
...
- job_name: 'node'
...
static_configs:
# Add node-exporter targets to be monitored below
- targets: [example-host:9100]
labels:
instance: 'my-host'
```
* Blackbox (in-stack exporter): update the `blackbox` job to add any endpoints to be monitored on the Blackbox dashboard:
```yml
...
@@ -85,7 +101,7 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de
- <HTTP_ENDPOINT_2>
```
* Postgres:
* Postgres (in-stack exporter):
* Update the `postgres` job to add Postgres db targets to be monitored:
```yml
@@ -111,6 +127,8 @@ Place the dashboard json files in grafana dashboards config directory (`monitori
Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`):
```bash
# For chain-head exporter
# External ETH RPC endpoint (ethereum)
# (Optional, default: https://mainnet.infura.io/v3)
CERC_ETH_RPC_ENDPOINT=
@@ -1,6 +1,6 @@
# Monitoring Watchers
Instructions to setup and run monitoring stack with pre-configured watcher dashboards
Instructions to setup and run monitoring stack with configured Watchers dashboard
## Create a deployment
@@ -35,6 +35,8 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-watchers-spec
## Configure
### Prometheus scrape config
Add the following scrape configs to prometheus config file (`monitoring-watchers-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder:
```yml
@@ -98,7 +100,28 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
chain: 'filecoin'
```
Add scrape config as done above for any additional watcher to add it to the watcher dashboard.
Add scrape config as done above for any additional watcher to add it to the Watchers dashboard.
### Grafana alerts config
Place the pre-configured watcher alerts rules in Grafana provisioning directory:
```bash
cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
```
Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points
Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/monitoring/grafana/provisioning/alerting/policies.yaml`) with appropriate object-matchers:
```yml
...
routes:
- receiver: SlackNotifier
object_matchers:
# Add matchers below
- ['grafana_folder', '=', 'WatcherAlerts']
```
### Env
@@ -0,0 +1,17 @@
# node-exporter
## Start the stack
```bash
laconic-so --stack node-exporter deploy up
```
* The host node's metrics can be accessed at `http://localhost:9100/metrics`
## Clean up
Stop the node-exporter running in background:
```bash
laconic-so --stack node-exporter deploy down
```
@@ -0,0 +1,6 @@
version: "0.1"
name: node-exporter
repos:
containers:
pods:
- node-exporter
@@ -2,9 +2,11 @@ version: "1.1"
name: package-registry
description: "Local Package Registry"
repos:
- git.vdb.to/cerc-io/hosting
- gitea.com/gitea/act_runner
- github.com/telackey/gitea@telackey/entrypoint
- git.vdb.to/cerc-io/hosting@dboreham/fix-dind
- gitea.com/telackey/act_runner@telackey/entrypoint
containers:
- cerc/gitea
- cerc/act-runner
- cerc/act-runner-task-executor
pods:
@@ -2,7 +2,7 @@ version: "1.0"
name: sushiswap-v3
description: "SushiSwap v3 watcher stack"
repos:
- github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.5
- github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.6
containers:
- cerc/watcher-sushiswap-v3
pods:
@@ -444,7 +444,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
deployer_config_generator = getDeployerConfigGenerator(deployment_type)
# TODO: make deployment_dir_path a Path above
deployer_config_generator.generate(deployment_dir_path)
call_stack_deploy_create(deployment_context, [network_dir, initial_peers])
call_stack_deploy_create(deployment_context, [network_dir, initial_peers, deployment_command_context])
# TODO: this code should be in the stack .py files but
@@ -26,7 +26,8 @@ from stack_orchestrator.deploy.webapp import deploy_webapp
from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
build_container_image, push_container_image,
file_hash, deploy_to_k8s, publish_deployment,
hostname_for_deployment_request, generate_hostname_for_app)
hostname_for_deployment_request, generate_hostname_for_app,
match_owner)
def process_app_deployment_request(
@@ -57,19 +58,12 @@ def process_app_deployment_request(
dns_crn = f"{dns_record_namespace}/{fqdn}"
dns_record = laconic.get_record(dns_crn)
if dns_record:
dns_record_owners = dns_record.owners
dns_request_owners = []
if dns_record.request:
prev_request = laconic.get_record(dns_record.request, require=True)
dns_request_owners = prev_request.owners
matched_owner = match_owner(app_deployment_request, dns_record)
if not matched_owner and dns_record.request:
matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.request, require=True))
owner_match = None
for owner in app_deployment_request.owners:
if owner in dns_request_owners or owner in dns_record_owners:
owner_match = owner
if owner_match:
print("Matched DnsRecord ownership to", owner)
if matched_owner:
print("Matched DnsRecord ownership:", matched_owner)
else:
raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" %
(dns_record.id, app_deployment_request.id))
@@ -153,7 +147,6 @@ def dump_known_requests(filename, requests):
@click.command()
@click.option("--kube-config", help="Provide a config file for a k8s deployment")
@click.option("--kube-config", help="Provide a config file for a k8s deployment")
@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster")
@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True)
@@ -237,9 +230,20 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
if d.attributes.request:
deployments_by_request[d.attributes.request] = d
# Find removal requests.
cancellation_requests = {}
removal_requests = laconic.app_deployment_removal_requests()
for r in removal_requests:
if r.attributes.request:
cancellation_requests[r.attributes.request] = r
requests_to_execute = []
for r in requests_by_name.values():
if r.id not in deployments_by_request:
if r.id in cancellation_requests and match_owner(cancellation_requests[r.id], r):
print(f"Found deployment cancellation request for {r.id} at {cancellation_requests[r.id].id}")
elif r.id in deployments_by_request:
print(f"Found satisfied request for {r.id} at {deployments_by_request[r.id].id}")
else:
if r.id not in previous_requests:
print(f"Request {r.id} needs to processed.")
requests_to_execute.append(r)
@@ -247,8 +251,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
print(
f"Skipping unsatisfied request {r.id} because we have seen it before."
)
else:
print(f"Found satisfied request {r.id} at {deployments_by_request[r.id].names[0]}")
print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))
@@ -0,0 +1,189 @@
# Copyright ©2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import json
import os
import shutil
import subprocess
import sys
import click
from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient, match_owner
def process_app_removal_request(ctx,
laconic: LaconicRegistryClient,
app_removal_request,
deployment_parent_dir,
delete_volumes,
delete_names):
deployment_record = laconic.get_record(app_removal_request.attributes.deployment, require=True)
dns_record = laconic.get_record(deployment_record.attributes.dns, require=True)
deployment_dir = os.path.join(deployment_parent_dir, dns_record.attributes.name)
if not os.path.exists(deployment_dir):
raise Exception("Deployment directory %s does not exist." % deployment_dir)
# Check if the removal request is from the owner of the DnsRecord or deployment record.
matched_owner = match_owner(app_removal_request, deployment_record, dns_record)
# Or of the original deployment request.
if not matched_owner and deployment_record.request:
matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.request, require=True))
if matched_owner:
print("Matched deployment ownership:", matched_owner)
else:
raise Exception("Unable to confirm ownership of deployment %s for removal request %s" %
(deployment_record.id, app_removal_request.id))
# TODO(telackey): Call the function directly. The easiest way to build the correct click context is to
# exec the process, but it would be better to refactor so we could just call down_operation with the
# necessary parameters
down_command = [sys.argv[0], "deployment", "--dir", deployment_dir, "down"]
if delete_volumes:
down_command.append("--delete-volumes")
result = subprocess.run(down_command)
result.check_returncode()
removal_record = {
"record": {
"type": "ApplicationDeploymentRemovalRecord",
"version": "1.0.0",
"request": app_removal_request.id,
"deployment": deployment_record.id,
}
}
laconic.publish(removal_record)
if delete_names:
if deployment_record.names:
for name in deployment_record.names:
laconic.delete_name(name)
if dns_record.names:
for name in dns_record.names:
laconic.delete_name(name)
if delete_volumes:
shutil.rmtree(deployment_dir)
def load_known_requests(filename):
if filename and os.path.exists(filename):
return json.load(open(filename, "r"))
return {}
def dump_known_requests(filename, requests):
if not filename:
return
known_requests = load_known_requests(filename)
for r in requests:
known_requests[r.id] = r.createTime
json.dump(known_requests, open(filename, "w"))
@click.command()
@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True)
@click.option("--request-id", help="The ApplicationDeploymentRemovalRequest to process")
@click.option("--discover", help="Discover and process all pending ApplicationDeploymentRemovalRequests",
is_flag=True, default=False)
@click.option("--state-file", help="File to store state about previously seen requests.")
@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True)
@click.option("--delete-names/--preserve-names", help="Delete all names associated with removed deployments.", default=True)
@click.option("--delete-volumes/--preserve-volumes", default=True, help="delete data volumes")
@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
@click.pass_context
def command(ctx, laconic_config, deployment_parent_dir,
request_id, discover, state_file, only_update_state,
delete_names, delete_volumes, dry_run):
if request_id and discover:
print("Cannot specify both --request-id and --discover", file=sys.stderr)
sys.exit(2)
if not request_id and not discover:
print("Must specify either --request-id or --discover", file=sys.stderr)
sys.exit(2)
if only_update_state and not state_file:
print("--only-update-state requires --state-file", file=sys.stderr)
sys.exit(2)
laconic = LaconicRegistryClient(laconic_config)
# Find deployment removal requests.
# single request
if request_id:
requests = [laconic.get_record(request_id, require=True)]
# TODO: assert record type
# all requests
elif discover:
requests = laconic.app_deployment_removal_requests()
if only_update_state:
if not dry_run:
dump_known_requests(state_file, requests)
return
previous_requests = load_known_requests(state_file)
requests.sort(key=lambda r: r.createTime)
# Find deployments.
deployments = {}
for d in laconic.app_deployments(all=True):
deployments[d.id] = d
# Find removal requests.
removals_by_deployment = {}
removals_by_request = {}
for r in laconic.app_deployment_removals():
if r.attributes.deployment:
# TODO: should we handle CRNs?
removals_by_deployment[r.attributes.deployment] = r
requests_to_execute = []
for r in requests:
if not r.attributes.deployment:
print(f"Skipping removal request {r.id} since it was a cancellation.")
elif r.id in removals_by_request:
print(f"Found satisfied request for {r.id} at {removals_by_request[r.id].id}")
elif r.attributes.deployment in removals_by_deployment:
print(
f"Found removal record for indicated deployment {r.attributes.deployment} at "
f"{removals_by_deployment[r.attributes.deployment].id}")
else:
if r.id not in previous_requests:
print(f"Request {r.id} needs to processed.")
requests_to_execute.append(r)
else:
print(f"Skipping unsatisfied request {r.id} because we have seen it before.")
print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))
if not dry_run:
for r in requests_to_execute:
try:
process_app_removal_request(
ctx,
laconic,
r,
os.path.abspath(deployment_parent_dir),
delete_volumes,
delete_names
)
finally:
dump_known_requests(state_file, [r])
+30 -8
View File
@@ -49,6 +49,14 @@ def cmd(*vargs):
raise err
def match_owner(recordA, *records):
for owner in recordA.owners:
for otherRecord in records:
if owner in otherRecord.owners:
return owner
return None
class LaconicRegistryClient:
def __init__(self, config_file):
self.config_file = config_file
@@ -146,11 +154,17 @@ class LaconicRegistryClient:
raise Exception("Cannot locate record:", name_or_id)
return None
def app_deployment_requests(self):
return self.list_records({"type": "ApplicationDeploymentRequest"}, True)
def app_deployment_requests(self, all=True):
return self.list_records({"type": "ApplicationDeploymentRequest"}, all)
def app_deployments(self):
return self.list_records({"type": "ApplicationDeploymentRecord"})
def app_deployments(self, all=True):
return self.list_records({"type": "ApplicationDeploymentRecord"}, all)
def app_deployment_removal_requests(self, all=True):
return self.list_records({"type": "ApplicationDeploymentRemovalRequest"}, all)
def app_deployment_removals(self, all=True):
return self.list_records({"type": "ApplicationDeploymentRemovalRecord"}, all)
def publish(self, record, names=[]):
tmpdir = tempfile.mkdtemp()
@@ -165,11 +179,17 @@ class LaconicRegistryClient:
cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname)
)["id"]
for name in names:
cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id)
self.set_name(name, new_record_id)
return new_record_id
finally:
cmd("rm", "-rf", tmpdir)
def set_name(self, name, record_id):
cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, record_id)
def delete_name(self, name):
cmd("laconic", "-c", self.config_file, "cns", "name", "delete", name)
def file_hash(filename):
return hashlib.sha1(open(filename).read().encode()).hexdigest()
@@ -187,9 +207,11 @@ def build_container_image(app_record, tag, extra_build_args=[]):
print(f"Cloning repository {repo} to {clone_dir} ...")
if ref:
# TODO: Determing branch or hash, and use depth 1 if we can.
result = subprocess.run(["git", "clone", repo, clone_dir])
result.check_returncode()
subprocess.check_call(["git", "checkout", ref], cwd=clone_dir)
git_env = dict(os.environ.copy())
# Never prompt
git_env["GIT_TERMINAL_PROMPT"] = "0"
subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env)
subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env)
else:
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir])
result.check_returncode()
+5 -1
View File
@@ -20,7 +20,10 @@ from stack_orchestrator.repos import setup_repositories
from stack_orchestrator.build import build_containers
from stack_orchestrator.build import build_npms
from stack_orchestrator.build import build_webapp
from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp, deploy_webapp_from_registry
from stack_orchestrator.deploy.webapp import (run_webapp,
deploy_webapp,
deploy_webapp_from_registry,
undeploy_webapp_from_registry)
from stack_orchestrator.deploy import deploy
from stack_orchestrator import version
from stack_orchestrator.deploy import deployment
@@ -54,6 +57,7 @@ cli.add_command(build_webapp.command, "build-webapp")
cli.add_command(run_webapp.command, "run-webapp")
cli.add_command(deploy_webapp.command, "deploy-webapp")
cli.add_command(deploy_webapp_from_registry.command, "deploy-webapp-from-registry")
cli.add_command(undeploy_webapp_from_registry.command, "undeploy-webapp-from-registry")
cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system
cli.add_command(deploy.command, "deploy-system")
cli.add_command(deployment.command, "deployment")