cerc-io/stack-orchestrator
Archived
12
1
Fork 3
Code Issues 117 Pull Requests 25 Actions Packages Projects Releases 493 Wiki Activity

Upgrade watchers and their config (#827)
Lint Checks / Run linter (push) Successful in 51s
Details
Publish / Build and publish (push) Successful in 1m28s
Details
Webapp Test / Run webapp test suite (push) Successful in 4m29s
Details
Smoke Test / Run basic test suite (push) Successful in 4m32s
Details
Deploy Test / Run deploy test suite (push) Successful in 5m26s
Details
Fixturenet-Laconicd-Test / Run Laconicd fixturenet and Laconic CLI tests (push) Successful in 22m46s
Details
K8s Deploy Test / Run deploy test suite on kind/k8s (push) Failing after 9m10s
Details
Fixturenet-Eth-Plugeth-Arm-Test / Run an Ethereum plugeth fixturenet test (push) Failing after 3h13m0s
Details
Fixturenet-Eth-Plugeth-Test / Run an Ethereum plugeth fixturenet test (push) Failing after 3h13m0s
Details
Database Test / Run database hosting test on kind/k8s (push) Successful in 18m47s
Details
Container Registry Test / Run contaier registry hosting test on kind/k8s (push) Successful in 5m12s
Details
External Stack Test / Run external stack test suite (push) Successful in 6m6s
Details

Browse Source 
Part of [Investigate subgraph watchers lagging behind head](https://www.notion.so/Investigate-subgraph-watchers-lagging-behind-head-01b72294ca8e4f658e4c0e86b36d19e2)

Co-authored-by: Nabarun <nabarun@deepstacksoft.com>
Reviewed-on: #827
Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
This commit was merged in pull request #827.
This commit is contained in:
prathamesh
2024-05-23 04:12:31 +00:00
committed by ashwin
co-authored by nabarun
parent fef7649683
commit ed8914b8d3
13 changed files with 353 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json
+273 -41
View File
@@ -1146,6 +1146,230 @@
"title": "DB size (MB)",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "dark-green",
"value": null
},
{
"color": "dark-orange",
"value": 8
},
{
"color": "dark-red",
"value": 16
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "diff_external"
},
"properties": [
{
"id": "color",
"value": {
"mode": "thresholds",
"seriesBy": "last"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "diff_external"
},
"properties": [
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "orange",
"value": 8
},
{
"color": "red",
"value": 16
}
]
}
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 12,
"x": 0,
"y": 12
},
"id": 33,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "10.2.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "latest_upstream_block_number{job=~\"$job\", instance=~\"$watcher\"}",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{{__name__}}",
"range": true,
"refId": "A",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}",
"hide": false,
"instant": false,
"legendFormat": "{{__name__}}",
"range": true,
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\"}",
"hide": false,
"instant": false,
"legendFormat": "{{__name__}}",
"range": true,
"refId": "C"
}
],
"title": "diff",
"transformations": [
{
"id": "calculateField",
"options": {
"alias": "diff_upstream",
"binary": {
"left": "latest_upstream_block_number",
"operator": "-",
"right": "sync_status_block_number"
},
"mode": "binary",
"reduce": {
"reducer": "sum"
},
"replaceFields": false
}
},
{
"id": "calculateField",
"options": {
"alias": "diff_external",
"binary": {
"left": "latest_block_number",
"operator": "-",
"right": "sync_status_block_number"
},
"mode": "binary",
"reduce": {
"reducer": "sum"
},
"replaceFields": false
}
},
{
"id": "organize",
"options": {
"excludeByName": {
"Time": false,
"latest_block_number": true,
"latest_upstream_block_number": true,
"sync_status_block_number": true
},
"includeByName": {},
"indexByName": {},
"renameByName": {}
}
}
],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
@@ -1208,8 +1432,8 @@
},
"gridPos": {
"h": 6,
"w": 12,
"x": 0,
"w": 6,
"x": 12,
"y": 12
},
"id": 5,
@@ -1310,8 +1534,8 @@
},
"gridPos": {
"h": 6,
"w": 12,
"x": 12,
"w": 6,
"x": 18,
"y": 12
},
"id": 15,
@@ -1367,17 +1591,28 @@
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 90,
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
@@ -1403,26 +1638,16 @@
},
"id": 28,
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": false,
"groupWidth": 1,
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"orientation": "auto",
"showValue": "auto",
"stacking": "none",
"tooltip": {
"mode": "single",
"sort": "none"
},
"xField": "Time",
"xTickLabelRotation": 0,
"xTickLabelSpacing": 0
}
},
"pluginVersion": "10.2.3",
"targets": [
@@ -1438,15 +1663,15 @@
"format": "time_series",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": true,
"instant": false,
"legendFormat": "{{method}}, {{provider}}",
"range": false,
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "ETH RPC request failures",
"type": "barchart"
"type": "timeseries"
},
{
"datasource": {
@@ -1561,17 +1786,28 @@
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 90,
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
@@ -1597,26 +1833,16 @@
},
"id": 30,
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": false,
"groupWidth": 1,
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"orientation": "auto",
"showValue": "auto",
"stacking": "none",
"tooltip": {
"mode": "single",
"sort": "none"
},
"xField": "Time",
"xTickLabelRotation": 0,
"xTickLabelSpacing": 0
}
},
"pluginVersion": "10.2.3",
"targets": [
@@ -1632,15 +1858,15 @@
"format": "time_series",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": true,
"instant": false,
"legendFormat": "{{method}}, {{provider}}",
"range": false,
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Total ETH RPC request failures (across all watchers)",
"type": "barchart"
"type": "timeseries"
},
{
"datasource": {
@@ -1805,7 +2031,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
}
]
}
@@ -1950,7 +2177,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
}
]
}
@@ -2097,7 +2325,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
}
]
}
@@ -2263,7 +2492,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
}
]
}
@@ -2360,7 +2590,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
}
]
}
@@ -2456,7 +2687,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
}
]
}
@@ -2610,6 +2842,6 @@
"timepicker": {},
"timezone": "",
"title": "Watchers",
"version": 2,
"version": 3,
"weekStart": ""
}
stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml
+2 -2
View File
@@ -67,9 +67,9 @@
isFEVM = true
# Boolean flag to filter event logs by contracts
filterLogsByAddresses = false
filterLogsByAddresses = true
# Boolean flag to filter event logs by topics
filterLogsByTopics = false
filterLogsByTopics = true
[upstream.cache]
name = "requests"
stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh
+7 -4
View File
@@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
# Replace env variables in template TOML file
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
# Write the modified content to a new file
stack_orchestrator/data/config/watcher-azimuth/start-server.sh
+7 -4
View File
@@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
# Replace env variables in template TOML file
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
# Write the modified content to a new file
stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml
+2 -2
View File
@@ -13,8 +13,8 @@
[upstream]
[upstream.ethServer]
gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL"
rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC"
gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT"
rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
[jobQueue]
historicalLogsBlockRange = REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE
stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml
+2 -2
View File
@@ -67,9 +67,9 @@
isFEVM = true
# Boolean flag to filter event logs by contracts
filterLogsByAddresses = false
filterLogsByAddresses = true
# Boolean flag to filter event logs by topics
filterLogsByTopics = false
filterLogsByTopics = true
[upstream.cache]
name = "requests"
stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml
+2 -2
View File
@@ -67,9 +67,9 @@
isFEVM = true
# Boolean flag to filter event logs by contracts
filterLogsByAddresses = false
filterLogsByAddresses = true
# Boolean flag to filter event logs by topics
filterLogsByTopics = false
filterLogsByTopics = true
[upstream.cache]
name = "requests"