cerc-io/stack-orchestrator
17
1
Fork 3
Code Issues 113 Pull Requests 18 Actions Packages Projects Releases 480 Wiki Activity

Update watcher stacks for backfilling event data #952

Open
prathamesh wants to merge 4 commits from deep-stack/stack-orchestrator:pm-upgrade-watcher-stacks into main
pull from: deep-stack/stack-orchestrator:pm-upgrade-watcher-stacks
merge into: cerc-io:main
Conversation 0 Commits 4 Files Changed 10 +66 -16
10 changed files with 66 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml
View File

@ -30,6 +30,8 @@ services:
environment: environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH:-/}
CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH:-/rpc}
command: ["bash", "./start-job-runner.sh"] command: ["bash", "./start-job-runner.sh"]
volumes: volumes:
- ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@ -56,6 +58,8 @@ services:
environment: environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH:-/}
CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH:-/rpc}
command: ["bash", "./start-server.sh"] command: ["bash", "./start-server.sh"]
volumes: volumes:
- ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml

4
stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml
View File

@ -30,6 +30,8 @@ services:
environment: environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH:-/}
CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH:-/rpc}
command: ["bash", "./start-job-runner.sh"] command: ["bash", "./start-job-runner.sh"]
volumes: volumes:
- ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@ -56,6 +58,8 @@ services:
environment: environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH:-/}
CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH:-/rpc}
command: ["bash", "./start-server.sh"] command: ["bash", "./start-server.sh"]
volumes: volumes:
- ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml

9
stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh
View File

@ -6,7 +6,10 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
fi fi
set -u set -u
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" echo "Using env variables:"
echo "CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}"
echo "CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH}"
echo "CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH}"
# Read in the config template TOML file and modify it # Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
@ -15,7 +18,9 @@ WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") sed -E "s|REPLACE_WITH_CERC_GQL_SERVER_PATH|\"${CERC_GQL_SERVER_PATH}\"|g; \
s|REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH|\"${CERC_ETH_RPC_SERVER_PATH}\"|g; \
s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
# Write the modified content to a new file # Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml echo "$WATCHER_CONFIG" > environments/local.toml

9
stack_orchestrator/data/config/watcher-ajna/start-server.sh
View File

@ -6,7 +6,10 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
fi fi
set -u set -u
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" echo "Using env variables:"
echo "CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}"
echo "CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH}"
echo "CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH}"
# Read in the config template TOML file and modify it # Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
@ -15,7 +18,9 @@ WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") sed -E "s|REPLACE_WITH_CERC_GQL_SERVER_PATH|\"${CERC_GQL_SERVER_PATH}\"|g; \
s|REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH|\"${CERC_ETH_RPC_SERVER_PATH}\"|g; \
s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
# Write the modified content to a new file # Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml echo "$WATCHER_CONFIG" > environments/local.toml

17
stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml
View File

@ -10,7 +10,6 @@
checkpointInterval = 2000 checkpointInterval = 2000
# Enable state creation # Enable state creation
# CAUTION: Disable only if state creation is not desired or can be filled subsequently
enableState = false enableState = false
subgraphPath = "./subgraph-build" subgraphPath = "./subgraph-build"
@ -24,9 +23,9 @@
# Flag to specify whether RPC endpoint supports block hash as block tag parameter # Flag to specify whether RPC endpoint supports block hash as block tag parameter
rpcSupportsBlockHashParam = false rpcSupportsBlockHashParam = false
# Server GQL config # GQL server config
[server.gql] [server.gql]
path = "/" path = REPLACE_WITH_CERC_GQL_SERVER_PATH
# Max block range for which to return events in eventsInRange GQL query. # Max block range for which to return events in eventsInRange GQL query.
# Use -1 for skipping check on block range. # Use -1 for skipping check on block range.
@ -46,6 +45,14 @@
maxAge = 15 maxAge = 15
timeTravelMaxAge = 86400 # 1 day timeTravelMaxAge = 86400 # 1 day
# ETH RPC server config
[server.ethRPC]
enabled = true
path = REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH
# Max number of logs that can be returned in a single getLogs request (default: 10000)
getLogsResultLimit = 10000
[metrics] [metrics]
host = "0.0.0.0" host = "0.0.0.0"
port = 9000 port = 9000
@ -77,6 +84,10 @@
# Boolean flag to filter event logs by topics # Boolean flag to filter event logs by topics
filterLogsByTopics = true filterLogsByTopics = true
# Switch clients if eth_getLogs call takes more than threshold (in secs)
# Set to 0 for disabling switching
getLogsClientSwitchThresholdInSecs = 30
[upstream.cache] [upstream.cache]
name = "requests" name = "requests"
enabled = false enabled = false

9
stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh
View File

@ -6,7 +6,10 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
fi fi
set -u set -u
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" echo "Using env variables:"
echo "CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}"
echo "CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH}"
echo "CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH}"
# Read in the config template TOML file and modify it # Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
@ -15,7 +18,9 @@ WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") sed -E "s|REPLACE_WITH_CERC_GQL_SERVER_PATH|\"${CERC_GQL_SERVER_PATH}\"|g; \
s|REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH|\"${CERC_ETH_RPC_SERVER_PATH}\"|g; \
s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
# Write the modified content to a new file # Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml echo "$WATCHER_CONFIG" > environments/local.toml

9
stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh
View File

@ -6,7 +6,10 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
fi fi
set -u set -u
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" echo "Using env variables:"
echo "CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}"
echo "CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH}"
echo "CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH}"
# Read in the config template TOML file and modify it # Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
@ -15,7 +18,9 @@ WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") sed -E "s|REPLACE_WITH_CERC_GQL_SERVER_PATH|\"${CERC_GQL_SERVER_PATH}\"|g; \
s|REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH|\"${CERC_ETH_RPC_SERVER_PATH}\"|g; \
s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
# Write the modified content to a new file # Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml echo "$WATCHER_CONFIG" > environments/local.toml

17
stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml
View File

@ -10,7 +10,6 @@
checkpointInterval = 2000 checkpointInterval = 2000
# Enable state creation # Enable state creation
# CAUTION: Disable only if state creation is not desired or can be filled subsequently
enableState = false enableState = false
subgraphPath = "./subgraph-build" subgraphPath = "./subgraph-build"
@ -24,9 +23,9 @@
# Flag to specify whether RPC endpoint supports block hash as block tag parameter # Flag to specify whether RPC endpoint supports block hash as block tag parameter
rpcSupportsBlockHashParam = false rpcSupportsBlockHashParam = false
# Server GQL config # GQL server config
[server.gql] [server.gql]
path = "/" path = REPLACE_WITH_CERC_GQL_SERVER_PATH
# Max block range for which to return events in eventsInRange GQL query. # Max block range for which to return events in eventsInRange GQL query.
# Use -1 for skipping check on block range. # Use -1 for skipping check on block range.
@ -46,6 +45,14 @@
maxAge = 15 maxAge = 15
timeTravelMaxAge = 86400 # 1 day timeTravelMaxAge = 86400 # 1 day
# ETH RPC server config
[server.ethRPC]
enabled = true
path = REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH
# Max number of logs that can be returned in a single getLogs request (default: 10000)
getLogsResultLimit = 10000
[metrics] [metrics]
host = "0.0.0.0" host = "0.0.0.0"
port = 9000 port = 9000
@ -77,6 +84,10 @@
# Boolean flag to filter event logs by topics # Boolean flag to filter event logs by topics
filterLogsByTopics = true filterLogsByTopics = true
# Switch clients if eth_getLogs call takes more than threshold (in secs)
# Set to 0 for disabling switching
getLogsClientSwitchThresholdInSecs = 30
[upstream.cache] [upstream.cache]
name = "requests" name = "requests"
enabled = false enabled = false

2
stack_orchestrator/data/stacks/ajna/stack.yml
View File

@ -2,7 +2,7 @@ version: "1.0"
name: ajna name: ajna
description: "Ajna watcher stack" description: "Ajna watcher stack"
repos: repos:
- git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.13 - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.17
containers: containers:
- cerc/watcher-ajna - cerc/watcher-ajna
pods: pods:

2
stack_orchestrator/data/stacks/sushiswap-v3/stack.yml
View File

@ -2,7 +2,7 @@ version: "1.0"
name: sushiswap-v3 name: sushiswap-v3
description: "SushiSwap v3 watcher stack" description: "SushiSwap v3 watcher stack"
repos: repos:
- github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.14 - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.17
containers: containers:
- cerc/watcher-sushiswap-v3 - cerc/watcher-sushiswap-v3
pods: pods: