Update sushiswap-v3 watcher scripts

stack_orchestrator/build/fetch_containers.py

@@ -14,7 +14,6 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 import click
-import os
 from dataclasses import dataclass
 import json
 import platform
@@ -85,18 +84,13 @@ def _filter_for_platform(container: str,
                          tag_list: List[str]) -> List[str] :
     filtered_tags = []
     this_machine = platform.machine()
-    if "DOCKER_DEFAULT_PLATFORM" in os.environ:
-        this_machine =  os.environ["DOCKER_DEFAULT_PLATFORM"].split("/")[-1]
-        if opts.o.debug:
-            print(f"DOCKER_DEFAULT_PLATFORM says the architecture is: {this_machine}")
-    else:
-        # Translate between Python and docker platform names
-        if this_machine == "x86_64":
-            this_machine = "amd64"
-        if this_machine == "aarch64":
-            this_machine = "arm64"
-        if opts.o.debug:
-            print(f"Python says the architecture is: {this_machine}")
+    # Translate between Python and docker platform names
+    if this_machine == "x86_64":
+        this_machine = "amd64"
+    if this_machine == "aarch64":
+        this_machine = "arm64"
+    if opts.o.debug:
+        print(f"Python says the architecture is: {this_machine}")
     docker = DockerClient()
     for tag in tag_list:
         remote_tag = f"{registry_info.registry}/{container}:{tag}"

stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml

@@ -30,6 +30,8 @@ services:
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH:-/}
+      CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH:-/rpc}
     command: ["bash", "./start-job-runner.sh"]
     volumes:
       - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -56,6 +58,8 @@ services:
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH:-/}
+      CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH:-/rpc}
     command: ["bash", "./start-server.sh"]
     volumes:
       - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml

stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml

@@ -30,6 +30,8 @@ services:
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH:-/}
+      CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH:-/rpc}
     command: ["bash", "./start-job-runner.sh"]
     volumes:
       - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -56,6 +58,8 @@ services:
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH:-/}
+      CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH:-/rpc}
     command: ["bash", "./start-server.sh"]
     volumes:
       - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml

stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh

@@ -6,7 +6,10 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using env variables:"
+echo "CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}"
+echo "CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH}"
+echo "CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
@@ -15,7 +18,9 @@ WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
 
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
+  sed -E "s|REPLACE_WITH_CERC_GQL_SERVER_PATH|\"${CERC_GQL_SERVER_PATH}\"|g; \
+          s|REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH|\"${CERC_ETH_RPC_SERVER_PATH}\"|g; \
+          s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-ajna/start-server.sh

@@ -6,7 +6,10 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using env variables:"
+echo "CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}"
+echo "CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH}"
+echo "CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
@@ -15,7 +18,9 @@ WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
 
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
+  sed -E "s|REPLACE_WITH_CERC_GQL_SERVER_PATH|\"${CERC_GQL_SERVER_PATH}\"|g; \
+          s|REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH|\"${CERC_ETH_RPC_SERVER_PATH}\"|g; \
+          s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml

@@ -10,7 +10,6 @@
   checkpointInterval = 2000
 
   # Enable state creation
-  # CAUTION: Disable only if state creation is not desired or can be filled subsequently
   enableState = false
 
   subgraphPath = "./subgraph-build"
@@ -24,9 +23,9 @@
   # Flag to specify whether RPC endpoint supports block hash as block tag parameter
   rpcSupportsBlockHashParam = false
 
-  # Server GQL config
+  # GQL server config
   [server.gql]
-    path = "/"
+    path = REPLACE_WITH_CERC_GQL_SERVER_PATH
 
     # Max block range for which to return events in eventsInRange GQL query.
     # Use -1 for skipping check on block range.
@@ -46,6 +45,14 @@
       maxAge = 15
       timeTravelMaxAge = 86400 # 1 day
 
+  # ETH RPC server config
+  [server.ethRPC]
+    enabled = true
+    path = REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH
+
+    # Max number of logs that can be returned in a single getLogs request (default: 10000)
+    getLogsResultLimit = 10000
+
 [metrics]
   host = "0.0.0.0"
   port = 9000
@@ -77,6 +84,10 @@
     # Boolean flag to filter event logs by topics
     filterLogsByTopics = true
 
+    # Switch clients if eth_getLogs call takes more than threshold (in secs)
+    # Set to 0 for disabling switching
+    getLogsClientSwitchThresholdInSecs = 30
+
   [upstream.cache]
     name = "requests"
     enabled = false

stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh

@@ -6,7 +6,10 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using env variables:"
+echo "CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}"
+echo "CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH}"
+echo "CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
@@ -15,7 +18,9 @@ WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
 
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
+  sed -E "s|REPLACE_WITH_CERC_GQL_SERVER_PATH|\"${CERC_GQL_SERVER_PATH}\"|g; \
+          s|REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH|\"${CERC_ETH_RPC_SERVER_PATH}\"|g; \
+          s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh

@@ -6,7 +6,10 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using env variables:"
+echo "CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}"
+echo "CERC_ETH_RPC_SERVER_PATH: ${CERC_ETH_RPC_SERVER_PATH}"
+echo "CERC_GQL_SERVER_PATH: ${CERC_GQL_SERVER_PATH}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
@@ -15,7 +18,9 @@ WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
 
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
+  sed -E "s|REPLACE_WITH_CERC_GQL_SERVER_PATH|\"${CERC_GQL_SERVER_PATH}\"|g; \
+          s|REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH|\"${CERC_ETH_RPC_SERVER_PATH}\"|g; \
+          s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml

@@ -10,7 +10,6 @@
   checkpointInterval = 2000
 
   # Enable state creation
-  # CAUTION: Disable only if state creation is not desired or can be filled subsequently
   enableState = false
 
   subgraphPath = "./subgraph-build"
@@ -24,9 +23,9 @@
   # Flag to specify whether RPC endpoint supports block hash as block tag parameter
   rpcSupportsBlockHashParam = false
 
-  # Server GQL config
+  # GQL server config
   [server.gql]
-    path = "/"
+    path = REPLACE_WITH_CERC_GQL_SERVER_PATH
 
     # Max block range for which to return events in eventsInRange GQL query.
     # Use -1 for skipping check on block range.
@@ -46,6 +45,14 @@
       maxAge = 15
       timeTravelMaxAge = 86400 # 1 day
 
+  # ETH RPC server config
+  [server.ethRPC]
+    enabled = true
+    path = REPLACE_WITH_CERC_ETH_RPC_SERVER_PATH
+
+    # Max number of logs that can be returned in a single getLogs request (default: 10000)
+    getLogsResultLimit = 10000
+
 [metrics]
   host = "0.0.0.0"
   port = 9000
@@ -77,6 +84,10 @@
     # Boolean flag to filter event logs by topics
     filterLogsByTopics = true
 
+    # Switch clients if eth_getLogs call takes more than threshold (in secs)
+    # Set to 0 for disabling switching
+    getLogsClientSwitchThresholdInSecs = 30
+
   [upstream.cache]
     name = "requests"
     enabled = false

stack_orchestrator/data/stacks/ajna/stack.yml

@@ -2,7 +2,7 @@ version: "1.0"
 name: ajna
 description: "Ajna watcher stack"
 repos:
-  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.13
+  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.17
 containers:
   - cerc/watcher-ajna
 pods:

stack_orchestrator/data/stacks/sushiswap-v3/stack.yml

@@ -2,7 +2,7 @@ version: "1.0"
 name: sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.14
+  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.17
 containers:
   - cerc/watcher-sushiswap-v3
 pods: