forked from cerc-io/stack-orchestrator
Upgrade watchers and their config (#827)
Part of [Investigate subgraph watchers lagging behind head](https://www.notion.so/Investigate-subgraph-watchers-lagging-behind-head-01b72294ca8e4f658e4c0e86b36d19e2) Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Reviewed-on: cerc-io/stack-orchestrator#827 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
This commit is contained in:
+273
-41
@@ -1146,6 +1146,230 @@
|
||||
"title": "DB size (MB)",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "dark-green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "dark-orange",
|
||||
"value": 8
|
||||
},
|
||||
{
|
||||
"color": "dark-red",
|
||||
"value": 16
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "diff_external"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "thresholds",
|
||||
"seriesBy": "last"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "diff_external"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "thresholds",
|
||||
"value": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "orange",
|
||||
"value": 8
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 16
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"id": 33,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.2.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"disableTextWrap": false,
|
||||
"editorMode": "code",
|
||||
"expr": "latest_upstream_block_number{job=~\"$job\", instance=~\"$watcher\"}",
|
||||
"fullMetaSearch": false,
|
||||
"hide": false,
|
||||
"includeNullMetadata": true,
|
||||
"instant": false,
|
||||
"legendFormat": "{{__name__}}",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"useBackend": false
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"legendFormat": "{{__name__}}",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\"}",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"legendFormat": "{{__name__}}",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"title": "diff",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "calculateField",
|
||||
"options": {
|
||||
"alias": "diff_upstream",
|
||||
"binary": {
|
||||
"left": "latest_upstream_block_number",
|
||||
"operator": "-",
|
||||
"right": "sync_status_block_number"
|
||||
},
|
||||
"mode": "binary",
|
||||
"reduce": {
|
||||
"reducer": "sum"
|
||||
},
|
||||
"replaceFields": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "calculateField",
|
||||
"options": {
|
||||
"alias": "diff_external",
|
||||
"binary": {
|
||||
"left": "latest_block_number",
|
||||
"operator": "-",
|
||||
"right": "sync_status_block_number"
|
||||
},
|
||||
"mode": "binary",
|
||||
"reduce": {
|
||||
"reducer": "sum"
|
||||
},
|
||||
"replaceFields": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {
|
||||
"Time": false,
|
||||
"latest_block_number": true,
|
||||
"latest_upstream_block_number": true,
|
||||
"sync_status_block_number": true
|
||||
},
|
||||
"includeByName": {},
|
||||
"indexByName": {},
|
||||
"renameByName": {}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
@@ -1208,8 +1432,8 @@
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 12
|
||||
},
|
||||
"id": 5,
|
||||
@@ -1310,8 +1534,8 @@
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 12
|
||||
},
|
||||
"id": 15,
|
||||
@@ -1367,17 +1591,28 @@
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"fillOpacity": 90,
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
@@ -1403,26 +1638,16 @@
|
||||
},
|
||||
"id": 28,
|
||||
"options": {
|
||||
"barRadius": 0,
|
||||
"barWidth": 0.97,
|
||||
"fullHighlight": false,
|
||||
"groupWidth": 1,
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"orientation": "auto",
|
||||
"showValue": "auto",
|
||||
"stacking": "none",
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
},
|
||||
"xField": "Time",
|
||||
"xTickLabelRotation": 0,
|
||||
"xTickLabelSpacing": 0
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.2.3",
|
||||
"targets": [
|
||||
@@ -1438,15 +1663,15 @@
|
||||
"format": "time_series",
|
||||
"fullMetaSearch": false,
|
||||
"includeNullMetadata": true,
|
||||
"instant": true,
|
||||
"instant": false,
|
||||
"legendFormat": "{{method}}, {{provider}}",
|
||||
"range": false,
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"useBackend": false
|
||||
}
|
||||
],
|
||||
"title": "ETH RPC request failures",
|
||||
"type": "barchart"
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
@@ -1561,17 +1786,28 @@
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"fillOpacity": 90,
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
@@ -1597,26 +1833,16 @@
|
||||
},
|
||||
"id": 30,
|
||||
"options": {
|
||||
"barRadius": 0,
|
||||
"barWidth": 0.97,
|
||||
"fullHighlight": false,
|
||||
"groupWidth": 1,
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"orientation": "auto",
|
||||
"showValue": "auto",
|
||||
"stacking": "none",
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
},
|
||||
"xField": "Time",
|
||||
"xTickLabelRotation": 0,
|
||||
"xTickLabelSpacing": 0
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.2.3",
|
||||
"targets": [
|
||||
@@ -1632,15 +1858,15 @@
|
||||
"format": "time_series",
|
||||
"fullMetaSearch": false,
|
||||
"includeNullMetadata": true,
|
||||
"instant": true,
|
||||
"instant": false,
|
||||
"legendFormat": "{{method}}, {{provider}}",
|
||||
"range": false,
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"useBackend": false
|
||||
}
|
||||
],
|
||||
"title": "Total ETH RPC request failures (across all watchers)",
|
||||
"type": "barchart"
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
@@ -1805,7 +2031,8 @@
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1950,7 +2177,8 @@
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -2097,7 +2325,8 @@
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -2263,7 +2492,8 @@
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -2360,7 +2590,8 @@
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -2456,7 +2687,8 @@
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -2610,6 +2842,6 @@
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "Watchers",
|
||||
"version": 2,
|
||||
"version": 3,
|
||||
"weekStart": ""
|
||||
}
|
||||
|
||||
@@ -67,9 +67,9 @@
|
||||
isFEVM = true
|
||||
|
||||
# Boolean flag to filter event logs by contracts
|
||||
filterLogsByAddresses = false
|
||||
filterLogsByAddresses = true
|
||||
# Boolean flag to filter event logs by topics
|
||||
filterLogsByTopics = false
|
||||
filterLogsByTopics = true
|
||||
|
||||
[upstream.cache]
|
||||
name = "requests"
|
||||
|
||||
@@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
|
||||
set -x
|
||||
fi
|
||||
|
||||
echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
|
||||
echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
|
||||
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
|
||||
echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
|
||||
echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
|
||||
|
||||
# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
|
||||
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
|
||||
|
||||
# Replace env variables in template TOML file
|
||||
# Read in the config template TOML file and modify it
|
||||
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
|
||||
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
|
||||
sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
|
||||
s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
|
||||
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
|
||||
s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
|
||||
s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
|
||||
|
||||
# Write the modified content to a new file
|
||||
|
||||
@@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
|
||||
set -x
|
||||
fi
|
||||
|
||||
echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
|
||||
echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
|
||||
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
|
||||
echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
|
||||
echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
|
||||
|
||||
# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
|
||||
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
|
||||
|
||||
# Replace env variables in template TOML file
|
||||
# Read in the config template TOML file and modify it
|
||||
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
|
||||
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
|
||||
sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
|
||||
s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
|
||||
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
|
||||
s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
|
||||
s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
|
||||
|
||||
# Write the modified content to a new file
|
||||
|
||||
@@ -13,8 +13,8 @@
|
||||
|
||||
[upstream]
|
||||
[upstream.ethServer]
|
||||
gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL"
|
||||
rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC"
|
||||
gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT"
|
||||
rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
|
||||
|
||||
[jobQueue]
|
||||
historicalLogsBlockRange = REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE
|
||||
|
||||
+2
-2
@@ -67,9 +67,9 @@
|
||||
isFEVM = true
|
||||
|
||||
# Boolean flag to filter event logs by contracts
|
||||
filterLogsByAddresses = false
|
||||
filterLogsByAddresses = true
|
||||
# Boolean flag to filter event logs by topics
|
||||
filterLogsByTopics = false
|
||||
filterLogsByTopics = true
|
||||
|
||||
[upstream.cache]
|
||||
name = "requests"
|
||||
|
||||
@@ -67,9 +67,9 @@
|
||||
isFEVM = true
|
||||
|
||||
# Boolean flag to filter event logs by contracts
|
||||
filterLogsByAddresses = false
|
||||
filterLogsByAddresses = true
|
||||
# Boolean flag to filter event logs by topics
|
||||
filterLogsByTopics = false
|
||||
filterLogsByTopics = true
|
||||
|
||||
[upstream.cache]
|
||||
name = "requests"
|
||||
|
||||
Reference in New Issue
Block a user