From 9c0680e5b411b3c8f79731084c56e331ff52211e Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Thu, 10 Oct 2024 18:23:07 +0530 Subject: [PATCH 1/4] Update sushiswap-v3 watcher config template --- .../watcher-config-template.toml | 15 +++++++++++++-- .../data/stacks/sushiswap-v3/stack.yml | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml index 817d5323..9af82df7 100644 --- a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml @@ -10,7 +10,6 @@ checkpointInterval = 2000 # Enable state creation - # CAUTION: Disable only if state creation is not desired or can be filled subsequently enableState = false subgraphPath = "./subgraph-build" @@ -24,7 +23,7 @@ # Flag to specify whether RPC endpoint supports block hash as block tag parameter rpcSupportsBlockHashParam = false - # Server GQL config + # GQL server config [server.gql] path = "/" @@ -46,6 +45,14 @@ maxAge = 15 timeTravelMaxAge = 86400 # 1 day + # ETH RPC server config + [server.ethRPC] + enabled = true + path = "/rpc" + + # Max number of logs that can be returned in a single getLogs request (default: 10000) + getLogsResultLimit = 10000 + [metrics] host = "0.0.0.0" port = 9000 @@ -77,6 +84,10 @@ # Boolean flag to filter event logs by topics filterLogsByTopics = true + # Switch clients if eth_getLogs call takes more than threshold (in secs) + # Set to 0 for disabling switching + getLogsClientSwitchThresholdInSecs = 30 + [upstream.cache] name = "requests" enabled = false diff --git a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml index ac29632b..9b6b7bb7 100644 --- a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.14 + - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.17 containers: - cerc/watcher-sushiswap-v3 pods: -- 2.45.2 From 4c60459d5c610e9496d5185639998dad88d63f27 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Mon, 14 Oct 2024 13:59:53 +0530 Subject: [PATCH 2/4] Update ajna watcher stack --- .../data/compose/docker-compose-watcher-ajna.yml | 4 ++++ .../data/config/watcher-ajna/start-job-runner.sh | 6 +++++- .../data/config/watcher-ajna/start-server.sh | 6 +++++- .../watcher-ajna/watcher-config-template.toml | 16 ++++++++++++++-- stack_orchestrator/data/stacks/ajna/stack.yml | 2 +- 5 files changed, 29 insertions(+), 5 deletions(-) diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml b/stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml index 84291ec4..043b404a 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml @@ -30,6 +30,8 @@ services: environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH:-/} + CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH:-/rpc} command: ["bash", "./start-job-runner.sh"] volumes: - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml @@ -56,6 +58,8 @@ services: environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH:-/} + CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH:-/rpc} command: ["bash", "./start-server.sh"] volumes: - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml diff --git a/stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh b/stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh index 7a7a83b5..59d43909 100755 --- a/stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh +++ b/stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh @@ -7,6 +7,8 @@ fi set -u echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" +echo "CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH}" +echo "CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH}" # Read in the config template TOML file and modify it WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) @@ -15,7 +17,9 @@ WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ - sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") + sed -E "s|REPLACE_WITH_CERC_GQL_SERVER_PATH|\"${CERC_GQL_SERVER_PATH}\"|g; \ + s|REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH|\"${CERC_ETH_RPC_SERVER_PATH}\"|g; \ + s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") # Write the modified content to a new file echo "$WATCHER_CONFIG" > environments/local.toml diff --git a/stack_orchestrator/data/config/watcher-ajna/start-server.sh b/stack_orchestrator/data/config/watcher-ajna/start-server.sh index 9aaa77ec..e0899964 100755 --- a/stack_orchestrator/data/config/watcher-ajna/start-server.sh +++ b/stack_orchestrator/data/config/watcher-ajna/start-server.sh @@ -7,6 +7,8 @@ fi set -u echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" +echo "CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH}" +echo "CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH}" # Read in the config template TOML file and modify it WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) @@ -15,7 +17,9 @@ WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ - sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") + sed -E "s|REPLACE_WITH_CERC_GQL_SERVER_PATH|\"${CERC_GQL_SERVER_PATH}\"|g; \ + s|REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH|\"${CERC_ETH_RPC_SERVER_PATH}\"|g; \ + s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") # Write the modified content to a new file echo "$WATCHER_CONFIG" > environments/local.toml diff --git a/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml index daa5238d..97038089 100644 --- a/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml @@ -24,9 +24,9 @@ # Flag to specify whether RPC endpoint supports block hash as block tag parameter rpcSupportsBlockHashParam = false - # Server GQL config + # GQL server config [server.gql] - path = "/" + path = REPLACE_WITH_CERC_GQL_SERVER_PATH # Max block range for which to return events in eventsInRange GQL query. # Use -1 for skipping check on block range. @@ -46,6 +46,14 @@ maxAge = 15 timeTravelMaxAge = 86400 # 1 day + # ETH RPC server config + [server.ethRPC] + enabled = true + path = REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH + + # Max number of logs that can be returned in a single getLogs request (default: 10000) + getLogsResultLimit = 10000 + [metrics] host = "0.0.0.0" port = 9000 @@ -77,6 +85,10 @@ # Boolean flag to filter event logs by topics filterLogsByTopics = true + # Switch clients if eth_getLogs call takes more than threshold (in secs) + # Set to 0 for disabling switching + getLogsClientSwitchThresholdInSecs = 30 + [upstream.cache] name = "requests" enabled = false diff --git a/stack_orchestrator/data/stacks/ajna/stack.yml b/stack_orchestrator/data/stacks/ajna/stack.yml index c6ba0c04..7ecfe5c7 100644 --- a/stack_orchestrator/data/stacks/ajna/stack.yml +++ b/stack_orchestrator/data/stacks/ajna/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: ajna description: "Ajna watcher stack" repos: - - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.13 + - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.17 containers: - cerc/watcher-ajna pods: -- 2.45.2 From efa7d12c2a32d384c744d01813ba1810592577dc Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Mon, 14 Oct 2024 17:06:49 +0530 Subject: [PATCH 3/4] Update sushiswap-v3 watcher stack --- .../data/compose/docker-compose-watcher-sushiswap-v3.yml | 4 ++++ .../data/config/watcher-ajna/start-job-runner.sh | 3 ++- stack_orchestrator/data/config/watcher-ajna/start-server.sh | 3 ++- .../data/config/watcher-ajna/watcher-config-template.toml | 1 - .../data/config/watcher-sushiswap-v3/start-job-runner.sh | 5 ++++- .../data/config/watcher-sushiswap-v3/start-server.sh | 5 ++++- .../config/watcher-sushiswap-v3/watcher-config-template.toml | 4 ++-- 7 files changed, 18 insertions(+), 7 deletions(-) diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml index 5cb6176f..499f36be 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml @@ -30,6 +30,8 @@ services: environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH:-/} + CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH:-/rpc} command: ["bash", "./start-job-runner.sh"] volumes: - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml @@ -56,6 +58,8 @@ services: environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH:-/} + CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH:-/rpc} command: ["bash", "./start-server.sh"] volumes: - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml diff --git a/stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh b/stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh index 59d43909..5e7f65fc 100755 --- a/stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh +++ b/stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh @@ -6,7 +6,8 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then fi set -u -echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" +echo "Using env variables:" +echo "CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}" echo "CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH}" echo "CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH}" diff --git a/stack_orchestrator/data/config/watcher-ajna/start-server.sh b/stack_orchestrator/data/config/watcher-ajna/start-server.sh index e0899964..f92dc9b0 100755 --- a/stack_orchestrator/data/config/watcher-ajna/start-server.sh +++ b/stack_orchestrator/data/config/watcher-ajna/start-server.sh @@ -6,7 +6,8 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then fi set -u -echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" +echo "Using env variables:" +echo "CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}" echo "CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH}" echo "CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH}" diff --git a/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml index 97038089..0e8057eb 100644 --- a/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml @@ -10,7 +10,6 @@ checkpointInterval = 2000 # Enable state creation - # CAUTION: Disable only if state creation is not desired or can be filled subsequently enableState = false subgraphPath = "./subgraph-build" diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh index 7a7a83b5..8be91dad 100755 --- a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh @@ -6,7 +6,10 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then fi set -u -echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" +echo "Using env variables:" +echo "CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}" +echo "CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH}" +echo "CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH}" # Read in the config template TOML file and modify it WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh index 9aaa77ec..b3f207fd 100755 --- a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh @@ -6,7 +6,10 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then fi set -u -echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" +echo "Using env variables:" +echo "CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}" +echo "CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH}" +echo "CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH}" # Read in the config template TOML file and modify it WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml index 9af82df7..596725eb 100644 --- a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml @@ -25,7 +25,7 @@ # GQL server config [server.gql] - path = "/" + path = REPLACE_WITH_CERC_GQL_SERVER_PATH # Max block range for which to return events in eventsInRange GQL query. # Use -1 for skipping check on block range. @@ -48,7 +48,7 @@ # ETH RPC server config [server.ethRPC] enabled = true - path = "/rpc" + path = REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH # Max number of logs that can be returned in a single getLogs request (default: 10000) getLogsResultLimit = 10000 -- 2.45.2 From 67d2e5897b8b92334731ff1cf4b85f0198f5dfa3 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Mon, 14 Oct 2024 17:40:15 +0530 Subject: [PATCH 4/4] Update sushiswap-v3 watcher scripts --- .../data/config/watcher-sushiswap-v3/start-job-runner.sh | 4 +++- .../data/config/watcher-sushiswap-v3/start-server.sh | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh index 8be91dad..5e7f65fc 100755 --- a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh @@ -18,7 +18,9 @@ WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ - sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") + sed -E "s|REPLACE_WITH_CERC_GQL_SERVER_PATH|\"${CERC_GQL_SERVER_PATH}\"|g; \ + s|REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH|\"${CERC_ETH_RPC_SERVER_PATH}\"|g; \ + s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") # Write the modified content to a new file echo "$WATCHER_CONFIG" > environments/local.toml diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh index b3f207fd..f92dc9b0 100755 --- a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh @@ -18,7 +18,9 @@ WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ - sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") + sed -E "s|REPLACE_WITH_CERC_GQL_SERVER_PATH|\"${CERC_GQL_SERVER_PATH}\"|g; \ + s|REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH|\"${CERC_ETH_RPC_SERVER_PATH}\"|g; \ + s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") # Write the modified content to a new file echo "$WATCHER_CONFIG" > environments/local.toml -- 2.45.2