From 4acb06325b349149d206e5de4dc859e5b77dfea0 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Wed, 12 Jun 2024 11:52:51 +0000 Subject: [PATCH] Update watcher dashboard and config templates (#835) Part of [Metrics and logging for GQL queries in watcher](https://www.notion.so/Metrics-and-logging-for-GQL-queries-in-watcher-928c692292b140a2a4f52cda9795df5e) - Update watcher config templates after config refactoring - Mount watcher GQL query log files on volumes - Update watcher dashboard to - add a panel to show latest processed block number - use latest processed block from sync status for diff values Co-authored-by: Nabarun Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/835 Co-authored-by: Prathamesh Musale Co-committed-by: Prathamesh Musale --- .../compose/docker-compose-watcher-ajna.yml | 2 + .../docker-compose-watcher-azimuth.yml | 16 +++ ...ker-compose-watcher-merkl-sushiswap-v3.yml | 2 + .../docker-compose-watcher-sushiswap-v3.yml | 2 + .../grafana/dashboards/watcher-dashboard.json | 97 ++++++++++++++++--- .../watcher-ajna/watcher-config-template.toml | 35 ++++--- .../watcher-config-template.toml | 3 +- .../watcher-config-template.toml | 35 ++++--- .../watcher-config-template.toml | 35 ++++--- stack_orchestrator/data/stacks/ajna/stack.yml | 2 +- .../data/stacks/merkl-sushiswap-v3/stack.yml | 2 +- .../data/stacks/sushiswap-v3/stack.yml | 2 +- 12 files changed, 176 insertions(+), 57 deletions(-) diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml b/stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml index 42381498..84291ec4 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml @@ -60,6 +60,7 @@ services: volumes: - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml - ../config/watcher-ajna/start-server.sh:/app/start-server.sh + - ajna_watcher_gql_logs_data:/app/gql-logs ports: - "3008" - "9001" @@ -74,3 +75,4 @@ services: volumes: ajna_watcher_db_data: + ajna_watcher_gql_logs_data: diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml b/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml index 3809113f..5bbac851 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml @@ -74,6 +74,7 @@ services: - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/azimuth-watcher/environments/watcher-config-template.toml - ../config/watcher-azimuth/merge-toml.js:/app/packages/azimuth-watcher/merge-toml.js - ../config/watcher-azimuth/start-server.sh:/app/packages/azimuth-watcher/start-server.sh + - azimuth_watcher_gql_logs_data:/app/packages/azimuth-watcher/gql-logs ports: - "3001" - "9001" @@ -136,6 +137,7 @@ services: - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/censures-watcher/environments/watcher-config-template.toml - ../config/watcher-azimuth/merge-toml.js:/app/packages/censures-watcher/merge-toml.js - ../config/watcher-azimuth/start-server.sh:/app/packages/censures-watcher/start-server.sh + - censures_watcher_gql_logs_data:/app/packages/censures-watcher/gql-logs ports: - "3002" - "9003" @@ -198,6 +200,7 @@ services: - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/claims-watcher/environments/watcher-config-template.toml - ../config/watcher-azimuth/merge-toml.js:/app/packages/claims-watcher/merge-toml.js - ../config/watcher-azimuth/start-server.sh:/app/packages/claims-watcher/start-server.sh + - claims_watcher_gql_logs_data:/app/packages/claims-watcher/gql-logs ports: - "3003" - "9005" @@ -260,6 +263,7 @@ services: - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/conditional-star-release-watcher/environments/watcher-config-template.toml - ../config/watcher-azimuth/merge-toml.js:/app/packages/conditional-star-release-watcher/merge-toml.js - ../config/watcher-azimuth/start-server.sh:/app/packages/conditional-star-release-watcher/start-server.sh + - conditional_star_release_watcher_gql_logs_data:/app/packages/conditional-star-release-watcher/gql-logs ports: - "3004" - "9007" @@ -322,6 +326,7 @@ services: - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/delegated-sending-watcher/environments/watcher-config-template.toml - ../config/watcher-azimuth/merge-toml.js:/app/packages/delegated-sending-watcher/merge-toml.js - ../config/watcher-azimuth/start-server.sh:/app/packages/delegated-sending-watcher/start-server.sh + - delegated_sending_watcher_gql_logs_data:/app/packages/delegated-sending-watcher/gql-logs ports: - "3005" - "9009" @@ -384,6 +389,7 @@ services: - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/ecliptic-watcher/environments/watcher-config-template.toml - ../config/watcher-azimuth/merge-toml.js:/app/packages/ecliptic-watcher/merge-toml.js - ../config/watcher-azimuth/start-server.sh:/app/packages/ecliptic-watcher/start-server.sh + - ecliptic_watcher_gql_logs_data:/app/packages/ecliptic-watcher/gql-logs ports: - "3006" - "9011" @@ -446,6 +452,7 @@ services: - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/linear-star-release-watcher/environments/watcher-config-template.toml - ../config/watcher-azimuth/merge-toml.js:/app/packages/linear-star-release-watcher/merge-toml.js - ../config/watcher-azimuth/start-server.sh:/app/packages/linear-star-release-watcher/start-server.sh + - linear_star_release_watcher_gql_logs_data:/app/packages/linear-star-release-watcher/gql-logs ports: - "3007" - "9013" @@ -508,6 +515,7 @@ services: - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/polls-watcher/environments/watcher-config-template.toml - ../config/watcher-azimuth/merge-toml.js:/app/packages/polls-watcher/merge-toml.js - ../config/watcher-azimuth/start-server.sh:/app/packages/polls-watcher/start-server.sh + - polls_watcher_gql_logs_data:/app/packages/polls-watcher/gql-logs ports: - "3008" - "9015" @@ -560,3 +568,11 @@ services: volumes: watcher_db_data: + azimuth_watcher_gql_logs_data: + censures_watcher_gql_logs_data: + claims_watcher_gql_logs_data: + conditional_star_release_watcher_gql_logs_data: + delegated_sending_watcher_gql_logs_data: + ecliptic_watcher_gql_logs_data: + linear_star_release_watcher_gql_logs_data: + polls_watcher_gql_logs_data: diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml index f7e505df..6a446b1f 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml @@ -60,6 +60,7 @@ services: volumes: - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh + - merkl_sushiswap_v3_watcher_gql_logs_data:/app/gql-logs ports: - "127.0.0.1:3007:3008" - "9003:9001" @@ -74,3 +75,4 @@ services: volumes: merkl_sushiswap_v3_watcher_db_data: + merkl_sushiswap_v3_watcher_gql_logs_data: diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml index 9aad9888..5cb6176f 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml @@ -60,6 +60,7 @@ services: volumes: - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh + - sushiswap_v3_watcher_gql_logs_data:/app/gql-logs ports: - "127.0.0.1:3008:3008" - "9001:9001" @@ -74,3 +75,4 @@ services: volumes: sushiswap_v3_watcher_db_data: + sushiswap_v3_watcher_gql_logs_data: diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json index 5e8998cd..b2582615 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 4, + "id": 2, "links": [ { "asDropdown": false, @@ -282,7 +282,7 @@ }, "gridPos": { "h": 3, - "w": 4, + "w": 3, "x": 0, "y": 4 }, @@ -303,6 +303,75 @@ "wideLayout": true }, "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_processed\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latest processed block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 4 + }, + "id": 38, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.3", "targets": [ { "datasource": { @@ -352,8 +421,8 @@ }, "gridPos": { "h": 3, - "w": 4, - "x": 4, + "w": 3, + "x": 6, "y": 4 }, "id": 11, @@ -421,8 +490,8 @@ }, "gridPos": { "h": 3, - "w": 4, - "x": 8, + "w": 3, + "x": 9, "y": 4 }, "id": 12, @@ -639,7 +708,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_processed\"}", "hide": false, "instant": false, "legendFormat": "{{__name__}}", @@ -750,7 +819,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_processed\"}", "hide": false, "instant": false, "legendFormat": "{{__name__}}", @@ -936,12 +1005,12 @@ "uid": "PBFA97CFB590B2093" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_processed_block_number{job=~\"$job\", instance=~\"$watcher\"}", + "editorMode": "code", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_processed\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "latest_processed", + "legendFormat": "{{kind}}", "range": true, "refId": "A", "useBackend": false @@ -952,7 +1021,7 @@ "uid": "PBFA97CFB590B2093" }, "disableTextWrap": false, - "editorMode": "builder", + "editorMode": "code", "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_canonical\"}", "fullMetaSearch": false, "hide": false, @@ -1395,7 +1464,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_processed\"}", "hide": false, "instant": false, "legendFormat": "{{__name__}}", @@ -3215,6 +3284,6 @@ "timepicker": {}, "timezone": "", "title": "Watchers", - "version": 4, + "version": 5, "weekStart": "" } diff --git a/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml index 8aca4e74..daa5238d 100644 --- a/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml @@ -2,7 +2,6 @@ host = "0.0.0.0" port = 3008 kind = "active" - gqlPath = "/" # Checkpointing state. checkpointing = true @@ -22,23 +21,30 @@ # Interval in number of blocks at which to clear entities cache. clearEntitiesCacheInterval = 1000 - # Max block range for which to return events in eventsInRange GQL query. - # Use -1 for skipping check on block range. - maxEventsBlockRange = 1000 - # Flag to specify whether RPC endpoint supports block hash as block tag parameter rpcSupportsBlockHashParam = false - # GQL cache settings - [server.gqlCache] - enabled = true + # Server GQL config + [server.gql] + path = "/" - # Max in-memory cache size (in bytes) (default 8 MB) - # maxCacheSize + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 - # GQL cache-control max-age settings (in seconds) - maxAge = 15 - timeTravelMaxAge = 86400 # 1 day + # Log directory for GQL requests + logDir = "./gql-logs" + + # GQL cache settings + [server.gql.cache] + enabled = true + + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize + + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day [metrics] host = "0.0.0.0" @@ -85,6 +91,9 @@ # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime blockDelayInMilliSecs = 30000 + # Number of blocks by which block processing lags behind head + blockProcessingOffset = 0 + # Boolean to switch between modes of processing events when starting the server. # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head). diff --git a/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml index 6ddaa6f6..40b7f80c 100644 --- a/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml @@ -1,6 +1,7 @@ [server] host = "0.0.0.0" - maxSimultaneousRequests = -1 + [server.gql] + maxSimultaneousRequests = -1 [metrics] host = "0.0.0.0" diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml index 33c7c22b..035843ff 100644 --- a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml @@ -2,7 +2,6 @@ host = "0.0.0.0" port = 3008 kind = "active" - gqlPath = '/' # Checkpointing state. checkpointing = true @@ -22,23 +21,30 @@ # Interval in number of blocks at which to clear entities cache. clearEntitiesCacheInterval = 1000 - # Max block range for which to return events in eventsInRange GQL query. - # Use -1 for skipping check on block range. - maxEventsBlockRange = 1000 - # Flag to specify whether RPC endpoint supports block hash as block tag parameter rpcSupportsBlockHashParam = false - # GQL cache settings - [server.gqlCache] - enabled = true + # Server GQL config + [server.gql] + path = "/" - # Max in-memory cache size (in bytes) (default 8 MB) - # maxCacheSize + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 - # GQL cache-control max-age settings (in seconds) - maxAge = 15 - timeTravelMaxAge = 86400 # 1 day + # Log directory for GQL requests + logDir = "./gql-logs" + + # GQL cache settings + [server.gql.cache] + enabled = true + + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize + + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day [metrics] host = "0.0.0.0" @@ -85,6 +91,9 @@ # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime blockDelayInMilliSecs = 30000 + # Number of blocks by which block processing lags behind head + blockProcessingOffset = 0 + # Boolean to switch between modes of processing events when starting the server. # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head). diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml index bbc19c26..817d5323 100644 --- a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml @@ -2,7 +2,6 @@ host = "0.0.0.0" port = 3008 kind = "active" - gqlPath = "/" # Checkpointing state. checkpointing = true @@ -22,23 +21,30 @@ # Interval in number of blocks at which to clear entities cache. clearEntitiesCacheInterval = 1000 - # Max block range for which to return events in eventsInRange GQL query. - # Use -1 for skipping check on block range. - maxEventsBlockRange = 1000 - # Flag to specify whether RPC endpoint supports block hash as block tag parameter rpcSupportsBlockHashParam = false - # GQL cache settings - [server.gqlCache] - enabled = true + # Server GQL config + [server.gql] + path = "/" - # Max in-memory cache size (in bytes) (default 8 MB) - # maxCacheSize + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 - # GQL cache-control max-age settings (in seconds) - maxAge = 15 - timeTravelMaxAge = 86400 # 1 day + # Log directory for GQL requests + logDir = "./gql-logs" + + # GQL cache settings + [server.gql.cache] + enabled = true + + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize + + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day [metrics] host = "0.0.0.0" @@ -85,6 +91,9 @@ # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime blockDelayInMilliSecs = 30000 + # Number of blocks by which block processing lags behind head + blockProcessingOffset = 0 + # Boolean to switch between modes of processing events when starting the server. # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head). diff --git a/stack_orchestrator/data/stacks/ajna/stack.yml b/stack_orchestrator/data/stacks/ajna/stack.yml index d5187984..c6ba0c04 100644 --- a/stack_orchestrator/data/stacks/ajna/stack.yml +++ b/stack_orchestrator/data/stacks/ajna/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: ajna description: "Ajna watcher stack" repos: - - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.12 + - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.13 containers: - cerc/watcher-ajna pods: diff --git a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml index c22633fc..779cb2e6 100644 --- a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: merkl-sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.13 + - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.14 containers: - cerc/watcher-merkl-sushiswap-v3 pods: diff --git a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml index 9e292847..ac29632b 100644 --- a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.13 + - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.14 containers: - cerc/watcher-sushiswap-v3 pods: