From ed8914b8d310646eed57a7614a9677f490af9165 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Thu, 23 May 2024 04:12:31 +0000 Subject: [PATCH] Upgrade watchers and their config (#827) Part of [Investigate subgraph watchers lagging behind head](https://www.notion.so/Investigate-subgraph-watchers-lagging-behind-head-01b72294ca8e4f658e4c0e86b36d19e2) Co-authored-by: Nabarun Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/827 Co-authored-by: Prathamesh Musale Co-committed-by: Prathamesh Musale --- .../docker-compose-watcher-azimuth.yml | 98 +++--- .../grafana/dashboards/watcher-dashboard.json | 314 +++++++++++++++--- .../watcher-ajna/watcher-config-template.toml | 4 +- .../watcher-azimuth/start-job-runner.sh | 11 +- .../config/watcher-azimuth/start-server.sh | 11 +- .../watcher-config-template.toml | 4 +- .../watcher-config-template.toml | 4 +- .../watcher-config-template.toml | 4 +- stack_orchestrator/data/stacks/ajna/stack.yml | 2 +- .../data/stacks/azimuth/README.md | 7 +- .../data/stacks/azimuth/stack.yml | 2 +- .../data/stacks/merkl-sushiswap-v3/stack.yml | 2 +- .../data/stacks/sushiswap-v3/stack.yml | 2 +- 13 files changed, 353 insertions(+), 112 deletions(-) diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml b/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml index 48e77082..cb25b561 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml @@ -32,8 +32,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} CERC_HISTORICAL_BLOCK_RANGE: 500 CONTRACT_ADDRESS: 0x223c067F8CF28ae173EE5CafEa60cA44C335fecB CONTRACT_NAME: Azimuth @@ -47,7 +47,7 @@ services: ports: - "9000" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "9000"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"] interval: 20s timeout: 5s retries: 15 @@ -66,8 +66,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} working_dir: /app/packages/azimuth-watcher command: "./start-server.sh" volumes: @@ -77,7 +77,7 @@ services: ports: - "3001" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "3001"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "3001"] interval: 20s timeout: 5s retries: 15 @@ -94,8 +94,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} CONTRACT_ADDRESS: 0x325f68d32BdEe6Ed86E7235ff2480e2A433D6189 CONTRACT_NAME: Censures STARTING_BLOCK: 6784954 @@ -108,7 +108,7 @@ services: ports: - "9002" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "9002"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "9002"] interval: 20s timeout: 5s retries: 15 @@ -127,8 +127,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} working_dir: /app/packages/censures-watcher command: "./start-server.sh" volumes: @@ -138,7 +138,7 @@ services: ports: - "3002" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "3002"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "3002"] interval: 20s timeout: 5s retries: 15 @@ -155,8 +155,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} CONTRACT_ADDRESS: 0xe7e7f69b34D7d9Bd8d61Fb22C33b22708947971A CONTRACT_NAME: Claims STARTING_BLOCK: 6784941 @@ -169,7 +169,7 @@ services: ports: - "9004" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "9004"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "9004"] interval: 20s timeout: 5s retries: 15 @@ -188,8 +188,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} working_dir: /app/packages/claims-watcher command: "./start-server.sh" volumes: @@ -199,7 +199,7 @@ services: ports: - "3003" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "3003"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "3003"] interval: 20s timeout: 5s retries: 15 @@ -216,8 +216,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} CONTRACT_ADDRESS: 0x8C241098C3D3498Fe1261421633FD57986D74AeA CONTRACT_NAME: ConditionalStarRelease STARTING_BLOCK: 6828004 @@ -230,7 +230,7 @@ services: ports: - "9006" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "9006"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "9006"] interval: 20s timeout: 5s retries: 15 @@ -249,8 +249,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} working_dir: /app/packages/conditional-star-release-watcher command: "./start-server.sh" volumes: @@ -260,7 +260,7 @@ services: ports: - "3004" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "3004"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "3004"] interval: 20s timeout: 5s retries: 15 @@ -277,8 +277,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} CONTRACT_ADDRESS: 0xf6b461fE1aD4bd2ce25B23Fe0aff2ac19B3dFA76 CONTRACT_NAME: DelegatedSending STARTING_BLOCK: 6784956 @@ -291,7 +291,7 @@ services: ports: - "9008" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "9008"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "9008"] interval: 20s timeout: 5s retries: 15 @@ -310,8 +310,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} working_dir: /app/packages/delegated-sending-watcher command: "./start-server.sh" volumes: @@ -321,7 +321,7 @@ services: ports: - "3005" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "3005"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "3005"] interval: 20s timeout: 5s retries: 15 @@ -338,8 +338,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} CONTRACT_ADDRESS: 0x33EeCbf908478C10614626A9D304bfe18B78DD73 CONTRACT_NAME: Ecliptic STARTING_BLOCK: 13692129 @@ -352,7 +352,7 @@ services: ports: - "9010" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "9010"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "9010"] interval: 20s timeout: 5s retries: 15 @@ -371,8 +371,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} working_dir: /app/packages/ecliptic-watcher command: "./start-server.sh" volumes: @@ -382,7 +382,7 @@ services: ports: - "3006" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "3006"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "3006"] interval: 20s timeout: 5s retries: 15 @@ -399,8 +399,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} CONTRACT_ADDRESS: 0x86cd9cd0992F04231751E3761De45cEceA5d1801 CONTRACT_NAME: LinearStarRelease STARTING_BLOCK: 6784943 @@ -413,7 +413,7 @@ services: ports: - "9012" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "9012"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "9012"] interval: 20s timeout: 5s retries: 15 @@ -432,8 +432,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} working_dir: /app/packages/linear-star-release-watcher command: "./start-server.sh" volumes: @@ -443,7 +443,7 @@ services: ports: - "3007" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "3007"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "3007"] interval: 20s timeout: 5s retries: 15 @@ -460,8 +460,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} CONTRACT_ADDRESS: 0x7fEcaB617c868Bb5996d99D95200D2Fa708218e4 CONTRACT_NAME: Polls STARTING_BLOCK: 6784912 @@ -474,7 +474,7 @@ services: ports: - "9014" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "9014"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "9014"] interval: 20s timeout: 5s retries: 15 @@ -493,8 +493,8 @@ services: condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} - CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT} working_dir: /app/packages/polls-watcher command: "./start-server.sh" volumes: @@ -504,7 +504,7 @@ services: ports: - "3008" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "3008"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"] interval: 20s timeout: 5s retries: 15 @@ -542,7 +542,7 @@ services: ports: - "0.0.0.0:4000:4000" healthcheck: - test: ["CMD", "nc", "-vz", "localhost", "4000"] + test: ["CMD", "nc", "-vz", "127.0.0.1", "4000"] interval: 20s timeout: 5s retries: 15 diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json index 0b62be5e..3a7771f4 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json @@ -1146,6 +1146,230 @@ "title": "DB size (MB)", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-orange", + "value": 8 + }, + { + "color": "dark-red", + "value": 16 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "diff_external" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "thresholds", + "seriesBy": "last" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "diff_external" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "C" + } + ], + "title": "diff", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "diff_upstream", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "calculateField", + "options": { + "alias": "diff_external", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": false, + "latest_block_number": true, + "latest_upstream_block_number": true, + "sync_status_block_number": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -1208,8 +1432,8 @@ }, "gridPos": { "h": 6, - "w": 12, - "x": 0, + "w": 6, + "x": 12, "y": 12 }, "id": 5, @@ -1310,8 +1534,8 @@ }, "gridPos": { "h": 6, - "w": 12, - "x": 12, + "w": 6, + "x": 18, "y": 12 }, "id": 15, @@ -1367,17 +1591,28 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "fillOpacity": 90, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, "thresholdsStyle": { "mode": "off" } @@ -1403,26 +1638,16 @@ }, "id": 28, "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 1, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", "tooltip": { "mode": "single", "sort": "none" - }, - "xField": "Time", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + } }, "pluginVersion": "10.2.3", "targets": [ @@ -1438,15 +1663,15 @@ "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, - "instant": true, + "instant": false, "legendFormat": "{{method}}, {{provider}}", - "range": false, + "range": true, "refId": "A", "useBackend": false } ], "title": "ETH RPC request failures", - "type": "barchart" + "type": "timeseries" }, { "datasource": { @@ -1561,17 +1786,28 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "fillOpacity": 90, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, "thresholdsStyle": { "mode": "off" } @@ -1597,26 +1833,16 @@ }, "id": 30, "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 1, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", "tooltip": { "mode": "single", "sort": "none" - }, - "xField": "Time", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + } }, "pluginVersion": "10.2.3", "targets": [ @@ -1632,15 +1858,15 @@ "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, - "instant": true, + "instant": false, "legendFormat": "{{method}}, {{provider}}", - "range": false, + "range": true, "refId": "A", "useBackend": false } ], "title": "Total ETH RPC request failures (across all watchers)", - "type": "barchart" + "type": "timeseries" }, { "datasource": { @@ -1805,7 +2031,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] } @@ -1950,7 +2177,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] } @@ -2097,7 +2325,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] } @@ -2263,7 +2492,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] } @@ -2360,7 +2590,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] } @@ -2456,7 +2687,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] } @@ -2610,6 +2842,6 @@ "timepicker": {}, "timezone": "", "title": "Watchers", - "version": 2, + "version": 3, "weekStart": "" } diff --git a/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml index 61c21f43..8aca4e74 100644 --- a/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml @@ -67,9 +67,9 @@ isFEVM = true # Boolean flag to filter event logs by contracts - filterLogsByAddresses = false + filterLogsByAddresses = true # Boolean flag to filter event logs by topics - filterLogsByTopics = false + filterLogsByTopics = true [upstream.cache] name = "requests" diff --git a/stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh b/stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh index 4bcad74c..f59ef6c6 100755 --- a/stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh +++ b/stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh @@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi -echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}" -echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}" +echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" +echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}" echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}" +# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array +RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') + # Replace env variables in template TOML file # Read in the config template TOML file and modify it WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ - sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \ - s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \ + s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \ s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ") # Write the modified content to a new file diff --git a/stack_orchestrator/data/config/watcher-azimuth/start-server.sh b/stack_orchestrator/data/config/watcher-azimuth/start-server.sh index fa334653..4e6bbf59 100755 --- a/stack_orchestrator/data/config/watcher-azimuth/start-server.sh +++ b/stack_orchestrator/data/config/watcher-azimuth/start-server.sh @@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi -echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}" -echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}" +echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" +echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}" echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}" +# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array +RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') + # Replace env variables in template TOML file # Read in the config template TOML file and modify it WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ - sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \ - s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \ + s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \ s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ") # Write the modified content to a new file diff --git a/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml index 2a91fedf..6ddaa6f6 100644 --- a/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml @@ -13,8 +13,8 @@ [upstream] [upstream.ethServer] - gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL" - rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC" + gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT" + rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS [jobQueue] historicalLogsBlockRange = REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml index 48877ae8..33c7c22b 100644 --- a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml @@ -67,9 +67,9 @@ isFEVM = true # Boolean flag to filter event logs by contracts - filterLogsByAddresses = false + filterLogsByAddresses = true # Boolean flag to filter event logs by topics - filterLogsByTopics = false + filterLogsByTopics = true [upstream.cache] name = "requests" diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml index 43278e69..bbc19c26 100644 --- a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml @@ -67,9 +67,9 @@ isFEVM = true # Boolean flag to filter event logs by contracts - filterLogsByAddresses = false + filterLogsByAddresses = true # Boolean flag to filter event logs by topics - filterLogsByTopics = false + filterLogsByTopics = true [upstream.cache] name = "requests" diff --git a/stack_orchestrator/data/stacks/ajna/stack.yml b/stack_orchestrator/data/stacks/ajna/stack.yml index 0e102c9e..27d3229e 100644 --- a/stack_orchestrator/data/stacks/ajna/stack.yml +++ b/stack_orchestrator/data/stacks/ajna/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: ajna description: "Ajna watcher stack" repos: - - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.6 + - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.9 containers: - cerc/watcher-ajna pods: diff --git a/stack_orchestrator/data/stacks/azimuth/README.md b/stack_orchestrator/data/stacks/azimuth/README.md index 21200369..b6f84e0b 100644 --- a/stack_orchestrator/data/stacks/azimuth/README.md +++ b/stack_orchestrator/data/stacks/azimuth/README.md @@ -4,7 +4,7 @@ Instructions to setup and deploy Azimuth Watcher stack ## Setup -Prerequisite: `ipld-eth-server` RPC and GQL endpoints +Prerequisite: External RPC endpoints Clone required repositories: @@ -94,7 +94,7 @@ Inside the deployment directory, open the file `config.env` and add variable to ```bash # External RPC endpoints - CERC_IPLD_ETH_RPC= + CERC_ETH_RPC_ENDPOINTS=https://example-rpc-endpoint-1,https://example-rpc-endpoint-2 ``` * NOTE: If RPC endpoint is on the host machine, use `host.docker.internal` as the hostname to access the host port, or use the `ip a` command to find the IP address of the `docker0` interface (this will usually be something like `172.17.0.1` or `172.18.0.1`) @@ -120,4 +120,7 @@ To stop all azimuth services and also delete data: ```bash laconic-so deployment --dir azimuth-deployment stop --delete-volumes + +# Remove deployment directory (deployment will have to be recreated for a re-run) +rm -r azimuth-deployment ``` diff --git a/stack_orchestrator/data/stacks/azimuth/stack.yml b/stack_orchestrator/data/stacks/azimuth/stack.yml index 374f2af0..34e950c4 100644 --- a/stack_orchestrator/data/stacks/azimuth/stack.yml +++ b/stack_orchestrator/data/stacks/azimuth/stack.yml @@ -1,7 +1,7 @@ version: "1.0" name: azimuth repos: - - github.com/cerc-io/azimuth-watcher-ts@v0.1.3 + - github.com/cerc-io/azimuth-watcher-ts@v0.1.4 containers: - cerc/watcher-azimuth pods: diff --git a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml index adc78d88..8c692c3d 100644 --- a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: merkl-sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.10 + - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.11 containers: - cerc/watcher-merkl-sushiswap-v3 pods: diff --git a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml index f258af24..6f213df8 100644 --- a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.10 + - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.11 containers: - cerc/watcher-sushiswap-v3 pods: