From a03267a44c99f79db5a4a93969ed4da6feba4279 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Thu, 13 Jun 2024 12:16:43 +0000 Subject: [PATCH 01/21] Add codegen configs used to generate watchers (#6) Part of [Generate watchers for sushiswap subgraphs deployed in graph-node](https://www.notion.so/Generate-watchers-for-sushiswap-subgraphs-deployed-in-graph-node-b3f2e475373d4ab1887d9f8720bd5ae6) Reviewed-on: https://git.vdb.to/cerc-io/sushiswap-watcher-ts/pulls/6 Co-authored-by: Prathamesh Musale Co-committed-by: Prathamesh Musale --- packages/blocks-watcher/codegen-config.yml | 29 +++++++++++++++++++ .../blocks-watcher/environments/local.toml | 3 +- .../codegen-config.yml | 29 +++++++++++++++++++ .../environments/local.toml | 3 +- packages/sushiswap-watcher/codegen-config.yml | 29 +++++++++++++++++++ .../sushiswap-watcher/environments/local.toml | 3 +- packages/v2-watcher/codegen-config.yml | 29 +++++++++++++++++++ packages/v2-watcher/environments/local.toml | 3 +- packages/v3-watcher/codegen-config.yml | 29 +++++++++++++++++++ packages/v3-watcher/environments/local.toml | 3 +- 10 files changed, 150 insertions(+), 10 deletions(-) create mode 100644 packages/blocks-watcher/codegen-config.yml create mode 100644 packages/route-processor-watcher/codegen-config.yml create mode 100644 packages/sushiswap-watcher/codegen-config.yml create mode 100644 packages/v2-watcher/codegen-config.yml create mode 100644 packages/v3-watcher/codegen-config.yml diff --git a/packages/blocks-watcher/codegen-config.yml b/packages/blocks-watcher/codegen-config.yml new file mode 100644 index 0000000..a0c82b2 --- /dev/null +++ b/packages/blocks-watcher/codegen-config.yml @@ -0,0 +1,29 @@ +# Contracts to watch (required). +# Can pass empty array ([]) when using subgraphPath. +contracts: [] + +# Output folder path (logs output using `stdout` if not provided). +outputFolder: /home/user/cerc-io/sushiswap-watcher-ts/packages/blocks-watcher + +# Code generation mode [eth_call | storage | all | none] (default: none). +mode: none + +# Kind of watcher [lazy | active] (default: active). +kind: active + +# Watcher server port (default: 3008). +port: 3008 + +# Flatten the input contract file(s) [true | false] (default: true). +flatten: true + +# Config for subgraph +subgraph: + # Path to subgraph repo directory containing package.json + directory: /home/user/cerc-io/sushiswap-subgraphs/subgraphs/blocks + + # Package manager that is used in subgraph repo for dependencies + packageManager: pnpm + + # Path to subgraph manifest/config file + configFile: /home/user/cerc-io/sushiswap-subgraphs/subgraphs/blocks/subgraph.yaml diff --git a/packages/blocks-watcher/environments/local.toml b/packages/blocks-watcher/environments/local.toml index 3c8c1c5..1b63e38 100644 --- a/packages/blocks-watcher/environments/local.toml +++ b/packages/blocks-watcher/environments/local.toml @@ -10,8 +10,7 @@ checkpointInterval = 2000 # Enable state creation - # CAUTION: Disable only if state creation is not desired or can be filled subsequently - enableState = true + enableState = false subgraphPath = "./subgraph-build" diff --git a/packages/route-processor-watcher/codegen-config.yml b/packages/route-processor-watcher/codegen-config.yml new file mode 100644 index 0000000..5e98e46 --- /dev/null +++ b/packages/route-processor-watcher/codegen-config.yml @@ -0,0 +1,29 @@ +# Contracts to watch (required). +# Can pass empty array ([]) when using subgraphPath. +contracts: [] + +# Output folder path (logs output using `stdout` if not provided). +outputFolder: /home/user/cerc-io/sushiswap-watcher-ts/packages/route-processor-watcher + +# Code generation mode [eth_call | storage | all | none] (default: none). +mode: none + +# Kind of watcher [lazy | active] (default: active). +kind: active + +# Watcher server port (default: 3008). +port: 3008 + +# Flatten the input contract file(s) [true | false] (default: true). +flatten: true + +# Config for subgraph +subgraph: + # Path to subgraph repo directory containing package.json + directory: /home/user/cerc-io/sushiswap-subgraphs/subgraphs/route-processor + + # Package manager that is used in subgraph repo for dependencies + packageManager: pnpm + + # Path to subgraph manifest/config file + configFile: /home/user/cerc-io/sushiswap-subgraphs/subgraphs/route-processor/subgraph.yaml diff --git a/packages/route-processor-watcher/environments/local.toml b/packages/route-processor-watcher/environments/local.toml index 6c7b896..d839ca4 100644 --- a/packages/route-processor-watcher/environments/local.toml +++ b/packages/route-processor-watcher/environments/local.toml @@ -10,8 +10,7 @@ checkpointInterval = 2000 # Enable state creation - # CAUTION: Disable only if state creation is not desired or can be filled subsequently - enableState = true + enableState = false subgraphPath = "./subgraph-build" diff --git a/packages/sushiswap-watcher/codegen-config.yml b/packages/sushiswap-watcher/codegen-config.yml new file mode 100644 index 0000000..d4dcacb --- /dev/null +++ b/packages/sushiswap-watcher/codegen-config.yml @@ -0,0 +1,29 @@ +# Contracts to watch (required). +# Can pass empty array ([]) when using subgraphPath. +contracts: [] + +# Output folder path (logs output using `stdout` if not provided). +outputFolder: /home/user/cerc-io/sushiswap-watcher-ts/packages/sushiswap-watcher + +# Code generation mode [eth_call | storage | all | none] (default: none). +mode: none + +# Kind of watcher [lazy | active] (default: active). +kind: active + +# Watcher server port (default: 3008). +port: 3008 + +# Flatten the input contract file(s) [true | false] (default: true). +flatten: true + +# Config for subgraph +subgraph: + # Path to subgraph repo directory containing package.json + directory: /home/user/cerc-io/sushiswap-subgraphs/subgraphs/sushiswap + + # Package manager that is used in subgraph repo for dependencies + packageManager: pnpm + + # Path to subgraph manifest/config file + configFile: /home/user/cerc-io/sushiswap-subgraphs/subgraphs/sushiswap/subgraph.yaml diff --git a/packages/sushiswap-watcher/environments/local.toml b/packages/sushiswap-watcher/environments/local.toml index c5baa72..969395b 100644 --- a/packages/sushiswap-watcher/environments/local.toml +++ b/packages/sushiswap-watcher/environments/local.toml @@ -10,8 +10,7 @@ checkpointInterval = 2000 # Enable state creation - # CAUTION: Disable only if state creation is not desired or can be filled subsequently - enableState = true + enableState = false subgraphPath = "./subgraph-build" diff --git a/packages/v2-watcher/codegen-config.yml b/packages/v2-watcher/codegen-config.yml new file mode 100644 index 0000000..0ab76ec --- /dev/null +++ b/packages/v2-watcher/codegen-config.yml @@ -0,0 +1,29 @@ +# Contracts to watch (required). +# Can pass empty array ([]) when using subgraphPath. +contracts: [] + +# Output folder path (logs output using `stdout` if not provided). +outputFolder: /home/user/cerc-io/sushiswap-watcher-ts/packages/v2-watcher + +# Code generation mode [eth_call | storage | all | none] (default: none). +mode: none + +# Kind of watcher [lazy | active] (default: active). +kind: active + +# Watcher server port (default: 3008). +port: 3008 + +# Flatten the input contract file(s) [true | false] (default: true). +flatten: true + +# Config for subgraph +subgraph: + # Path to subgraph repo directory containing package.json + directory: /home/user/cerc-io/sushiswap-subgraphs/subgraphs/v2 + + # Package manager that is used in subgraph repo for dependencies + packageManager: pnpm + + # Path to subgraph manifest/config file + configFile: /home/user/cerc-io/sushiswap-subgraphs/subgraphs/v2/subgraph.yaml diff --git a/packages/v2-watcher/environments/local.toml b/packages/v2-watcher/environments/local.toml index c91253a..e90b792 100644 --- a/packages/v2-watcher/environments/local.toml +++ b/packages/v2-watcher/environments/local.toml @@ -10,8 +10,7 @@ checkpointInterval = 2000 # Enable state creation - # CAUTION: Disable only if state creation is not desired or can be filled subsequently - enableState = true + enableState = false subgraphPath = "./subgraph-build" diff --git a/packages/v3-watcher/codegen-config.yml b/packages/v3-watcher/codegen-config.yml new file mode 100644 index 0000000..9abdc29 --- /dev/null +++ b/packages/v3-watcher/codegen-config.yml @@ -0,0 +1,29 @@ +# Contracts to watch (required). +# Can pass empty array ([]) when using subgraphPath. +contracts: [] + +# Output folder path (logs output using `stdout` if not provided). +outputFolder: /home/user/cerc-io/sushiswap-watcher-ts/packages/v3-watcher + +# Code generation mode [eth_call | storage | all | none] (default: none). +mode: none + +# Kind of watcher [lazy | active] (default: active). +kind: active + +# Watcher server port (default: 3008). +port: 3008 + +# Flatten the input contract file(s) [true | false] (default: true). +flatten: true + +# Config for subgraph +subgraph: + # Path to subgraph repo directory containing package.json + directory: /home/user/cerc-io/sushiswap-subgraphs/subgraphs/v3 + + # Package manager that is used in subgraph repo for dependencies + packageManager: pnpm + + # Path to subgraph manifest/config file + configFile: /home/user/cerc-io/sushiswap-subgraphs/subgraphs/v3/subgraph.yaml diff --git a/packages/v3-watcher/environments/local.toml b/packages/v3-watcher/environments/local.toml index 98254ca..7dd0580 100644 --- a/packages/v3-watcher/environments/local.toml +++ b/packages/v3-watcher/environments/local.toml @@ -10,8 +10,7 @@ checkpointInterval = 2000 # Enable state creation - # CAUTION: Disable only if state creation is not desired or can be filled subsequently - enableState = true + enableState = false subgraphPath = "./subgraph-build" From 60a78b5d6c9697c4ef8e3a8470fc764674bf32eb Mon Sep 17 00:00:00 2001 From: Nabarun Date: Thu, 13 Jun 2024 12:21:41 +0000 Subject: [PATCH 02/21] Add readme step to build watchers (#7) Part of [Generate watchers for sushiswap subgraphs deployed in graph-node](https://www.notion.so/Generate-watchers-for-sushiswap-subgraphs-deployed-in-graph-node-b3f2e475373d4ab1887d9f8720bd5ae6) Reviewed-on: https://git.vdb.to/cerc-io/sushiswap-watcher-ts/pulls/7 Co-authored-by: Nabarun Co-committed-by: Nabarun --- README.md | 10 ++++++++++ packages/blocks-watcher/README.md | 6 +----- packages/route-processor-watcher/README.md | 6 +----- packages/sushiswap-watcher/README.md | 6 +----- packages/v2-watcher/README.md | 6 +----- packages/v3-watcher/README.md | 6 +----- 6 files changed, 15 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 3fbe763..288447d 100644 --- a/README.md +++ b/README.md @@ -1 +1,11 @@ # sushiswap-watcher-ts + +Watchers for the [sushiswap-subgraphs](https://github.com/cerc-io/sushiswap-subgraphs/tree/watcher-ts) on Filecoin + +## Setup + +* Run the following command to install required packages: + + ```bash + yarn && yarn build + ``` diff --git a/packages/blocks-watcher/README.md b/packages/blocks-watcher/README.md index 095ecb1..366e6bd 100644 --- a/packages/blocks-watcher/README.md +++ b/packages/blocks-watcher/README.md @@ -7,11 +7,7 @@ ## Setup -* Run the following command to install required packages: - - ```bash - yarn - ``` +* Follow root [README.md setup](../../README.md#setup) * Create a postgres12 database for the watcher: diff --git a/packages/route-processor-watcher/README.md b/packages/route-processor-watcher/README.md index 1f4f259..676712d 100644 --- a/packages/route-processor-watcher/README.md +++ b/packages/route-processor-watcher/README.md @@ -7,11 +7,7 @@ ## Setup -* Run the following command to install required packages: - - ```bash - yarn - ``` +* Follow root [README.md setup](../../README.md#setup) * Create a postgres12 database for the watcher: diff --git a/packages/sushiswap-watcher/README.md b/packages/sushiswap-watcher/README.md index 9061fbc..92e1231 100644 --- a/packages/sushiswap-watcher/README.md +++ b/packages/sushiswap-watcher/README.md @@ -7,11 +7,7 @@ ## Setup -* Run the following command to install required packages: - - ```bash - yarn - ``` +* Follow root [README.md setup](../../README.md#setup) * Create a postgres12 database for the watcher: diff --git a/packages/v2-watcher/README.md b/packages/v2-watcher/README.md index 053bc27..d3eb1ad 100644 --- a/packages/v2-watcher/README.md +++ b/packages/v2-watcher/README.md @@ -7,11 +7,7 @@ ## Setup -* Run the following command to install required packages: - - ```bash - yarn - ``` +* Follow root [README.md setup](../../README.md#setup) * Create a postgres12 database for the watcher: diff --git a/packages/v3-watcher/README.md b/packages/v3-watcher/README.md index 368aed1..bb4742b 100644 --- a/packages/v3-watcher/README.md +++ b/packages/v3-watcher/README.md @@ -7,11 +7,7 @@ ## Setup -* Run the following command to install required packages: - - ```bash - yarn - ``` +* Follow root [README.md setup](../../README.md#setup) * Create a postgres12 database for the watcher: From b7c16df064cb481fc37b923d3549342d3135ed07 Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Thu, 13 Jun 2024 13:55:12 +0530 Subject: [PATCH 03/21] Add docker compose file for v3 --- .../compose/docker-compose-watcher-v3.yml | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 stacks/stack-orchestrator/compose/docker-compose-watcher-v3.yml diff --git a/stacks/stack-orchestrator/compose/docker-compose-watcher-v3.yml b/stacks/stack-orchestrator/compose/docker-compose-watcher-v3.yml new file mode 100644 index 0000000..2ffa663 --- /dev/null +++ b/stacks/stack-orchestrator/compose/docker-compose-watcher-v3.yml @@ -0,0 +1,78 @@ +version: '3.2' + +services: + v3-watcher-db: + restart: unless-stopped + image: postgres:14-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_MULTIPLE_DATABASES=v3-watcher,v3-watcher-job-queue + - POSTGRES_EXTENSION=v3-watcher-job-queue:pgcrypto + - POSTGRES_PASSWORD=password + volumes: + - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh + - v3_watcher_db_data:/var/lib/postgresql/data + ports: + - "5432" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + + v3-watcher-job-runner: + restart: unless-stopped + depends_on: + v3-watcher-db: + condition: service_healthy + image: cerc/watcher-v3:local + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + command: ["bash", "./start-job-runner.sh"] + volumes: + - ../config/watcher-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-v3/start-job-runner.sh:/app/start-job-runner.sh + ports: + - "9000:9000" + healthcheck: + test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + v3-watcher-server: + restart: unless-stopped + depends_on: + v3-watcher-db: + condition: service_healthy + v3-watcher-job-runner: + condition: service_healthy + image: cerc/watcher-v3:local + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + command: ["bash", "./start-server.sh"] + volumes: + - ../config/watcher-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-v3/start-server.sh:/app/start-server.sh + - v3_watcher_gql_logs_data:/app/gql-logs + ports: + - "127.0.0.1:3008:3008" + - "9001:9001" + healthcheck: + test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + v3_watcher_db_data: + v3_watcher_gql_logs_data: From e254db1e1124f8924b154385cce68846cccdfe0a Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Thu, 13 Jun 2024 14:02:06 +0530 Subject: [PATCH 04/21] Add config template and scripts for v3 watcher --- .../config/watcher-v3/start-job-runner.sh | 24 ++++ .../config/watcher-v3/start-server.sh | 24 ++++ .../watcher-v3/watcher-config-template.toml | 110 ++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100755 stacks/stack-orchestrator/config/watcher-v3/start-job-runner.sh create mode 100755 stacks/stack-orchestrator/config/watcher-v3/start-server.sh create mode 100644 stacks/stack-orchestrator/config/watcher-v3/watcher-config-template.toml diff --git a/stacks/stack-orchestrator/config/watcher-v3/start-job-runner.sh b/stacks/stack-orchestrator/config/watcher-v3/start-job-runner.sh new file mode 100755 index 0000000..7a7a83b --- /dev/null +++ b/stacks/stack-orchestrator/config/watcher-v3/start-job-runner.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +set -u + +echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) + +# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array +RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') + +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +echo "Running job-runner..." +DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js diff --git a/stacks/stack-orchestrator/config/watcher-v3/start-server.sh b/stacks/stack-orchestrator/config/watcher-v3/start-server.sh new file mode 100755 index 0000000..9aaa77e --- /dev/null +++ b/stacks/stack-orchestrator/config/watcher-v3/start-server.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +set -u + +echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) + +# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array +RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') + +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +echo "Running server..." +DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js diff --git a/stacks/stack-orchestrator/config/watcher-v3/watcher-config-template.toml b/stacks/stack-orchestrator/config/watcher-v3/watcher-config-template.toml new file mode 100644 index 0000000..817d532 --- /dev/null +++ b/stacks/stack-orchestrator/config/watcher-v3/watcher-config-template.toml @@ -0,0 +1,110 @@ +[server] + host = "0.0.0.0" + port = 3008 + kind = "active" + + # Checkpointing state. + checkpointing = true + + # Checkpoint interval in number of blocks. + checkpointInterval = 2000 + + # Enable state creation + # CAUTION: Disable only if state creation is not desired or can be filled subsequently + enableState = false + + subgraphPath = "./subgraph-build" + + # Interval to restart wasm instance periodically + wasmRestartBlocksInterval = 20 + + # Interval in number of blocks at which to clear entities cache. + clearEntitiesCacheInterval = 1000 + + # Flag to specify whether RPC endpoint supports block hash as block tag parameter + rpcSupportsBlockHashParam = false + + # Server GQL config + [server.gql] + path = "/" + + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 + + # Log directory for GQL requests + logDir = "./gql-logs" + + # GQL cache settings + [server.gql.cache] + enabled = true + + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize + + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day + +[metrics] + host = "0.0.0.0" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "sushiswap-v3-watcher-db" + port = 5432 + database = "sushiswap-v3-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + +[upstream] + [upstream.ethServer] + rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS + + # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client) + rpcClient = true + + # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint + isFEVM = true + + # Boolean flag to filter event logs by contracts + filterLogsByAddresses = true + # Boolean flag to filter event logs by topics + filterLogsByTopics = true + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@sushiswap-v3-watcher-db/sushiswap-v3-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 100 + eventsInBatch = 50 + subgraphEventsOrder = true + # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime + blockDelayInMilliSecs = 30000 + + # Number of blocks by which block processing lags behind head + blockProcessingOffset = 0 + + # Boolean to switch between modes of processing events when starting the server. + # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. + # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head). + useBlockRanges = true + + # Block range in which logs are fetched during historical blocks processing + historicalLogsBlockRange = 2000 + + # Max block range of historical processing after which it waits for completion of events processing + # If set to -1 historical processing does not wait for events processing and completes till latest canonical block + historicalMaxFetchAhead = 10000 + + # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints + maxNewBlockRetries = 3 From 408e12b7410acbdc3ac2ab1c364f9124663c26f8 Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Thu, 13 Jun 2024 14:45:05 +0530 Subject: [PATCH 05/21] Update stack name --- .../compose/docker-compose-watcher-v3.yml | 36 +++++++++---------- .../watcher-v3/watcher-config-template.toml | 35 +++++++----------- stacks/stack-orchestrator/stacks/v3/stack.yml | 8 +++++ 3 files changed, 39 insertions(+), 40 deletions(-) create mode 100644 stacks/stack-orchestrator/stacks/v3/stack.yml diff --git a/stacks/stack-orchestrator/compose/docker-compose-watcher-v3.yml b/stacks/stack-orchestrator/compose/docker-compose-watcher-v3.yml index 2ffa663..102e784 100644 --- a/stacks/stack-orchestrator/compose/docker-compose-watcher-v3.yml +++ b/stacks/stack-orchestrator/compose/docker-compose-watcher-v3.yml @@ -1,17 +1,17 @@ version: '3.2' services: - v3-watcher-db: + sushiswap-v3-watcher-db: restart: unless-stopped image: postgres:14-alpine environment: - POSTGRES_USER=vdbm - - POSTGRES_MULTIPLE_DATABASES=v3-watcher,v3-watcher-job-queue - - POSTGRES_EXTENSION=v3-watcher-job-queue:pgcrypto + - POSTGRES_MULTIPLE_DATABASES=sushiswap-v3-watcher,sushiswap-v3-watcher-job-queue + - POSTGRES_EXTENSION=sushiswap-v3-watcher-job-queue:pgcrypto - POSTGRES_PASSWORD=password volumes: - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh - - v3_watcher_db_data:/var/lib/postgresql/data + - sushiswap-v3_watcher_db_data:/var/lib/postgresql/data ports: - "5432" healthcheck: @@ -21,19 +21,19 @@ services: retries: 15 start_period: 10s - v3-watcher-job-runner: + sushiswap-v3-watcher-job-runner: restart: unless-stopped depends_on: - v3-watcher-db: + sushiswap-v3-watcher-db: condition: service_healthy - image: cerc/watcher-v3:local + image: cerc/watcher-sushiswap-v3:local environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} command: ["bash", "./start-job-runner.sh"] volumes: - - ../config/watcher-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml - - ../config/watcher-v3/start-job-runner.sh:/app/start-job-runner.sh + - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh ports: - "9000:9000" healthcheck: @@ -45,22 +45,22 @@ services: extra_hosts: - "host.docker.internal:host-gateway" - v3-watcher-server: + sushiswap-v3-watcher-server: restart: unless-stopped depends_on: - v3-watcher-db: + sushiswap-v3-watcher-db: condition: service_healthy - v3-watcher-job-runner: + sushiswap-v3-watcher-job-runner: condition: service_healthy - image: cerc/watcher-v3:local + image: cerc/watcher-sushiswap-v3:local environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} command: ["bash", "./start-server.sh"] volumes: - - ../config/watcher-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml - - ../config/watcher-v3/start-server.sh:/app/start-server.sh - - v3_watcher_gql_logs_data:/app/gql-logs + - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh + - sushiswap-v3_watcher_gql_logs_data:/app/gql-logs ports: - "127.0.0.1:3008:3008" - "9001:9001" @@ -74,5 +74,5 @@ services: - "host.docker.internal:host-gateway" volumes: - v3_watcher_db_data: - v3_watcher_gql_logs_data: + sushiswap-v3_watcher_db_data: + sushiswap-v3_watcher_gql_logs_data: diff --git a/stacks/stack-orchestrator/config/watcher-v3/watcher-config-template.toml b/stacks/stack-orchestrator/config/watcher-v3/watcher-config-template.toml index 817d532..bbc19c2 100644 --- a/stacks/stack-orchestrator/config/watcher-v3/watcher-config-template.toml +++ b/stacks/stack-orchestrator/config/watcher-v3/watcher-config-template.toml @@ -2,6 +2,7 @@ host = "0.0.0.0" port = 3008 kind = "active" + gqlPath = "/" # Checkpointing state. checkpointing = true @@ -21,30 +22,23 @@ # Interval in number of blocks at which to clear entities cache. clearEntitiesCacheInterval = 1000 + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 + # Flag to specify whether RPC endpoint supports block hash as block tag parameter rpcSupportsBlockHashParam = false - # Server GQL config - [server.gql] - path = "/" + # GQL cache settings + [server.gqlCache] + enabled = true - # Max block range for which to return events in eventsInRange GQL query. - # Use -1 for skipping check on block range. - maxEventsBlockRange = 1000 + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize - # Log directory for GQL requests - logDir = "./gql-logs" - - # GQL cache settings - [server.gql.cache] - enabled = true - - # Max in-memory cache size (in bytes) (default 8 MB) - # maxCacheSize - - # GQL cache-control max-age settings (in seconds) - maxAge = 15 - timeTravelMaxAge = 86400 # 1 day + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day [metrics] host = "0.0.0.0" @@ -91,9 +85,6 @@ # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime blockDelayInMilliSecs = 30000 - # Number of blocks by which block processing lags behind head - blockProcessingOffset = 0 - # Boolean to switch between modes of processing events when starting the server. # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head). diff --git a/stacks/stack-orchestrator/stacks/v3/stack.yml b/stacks/stack-orchestrator/stacks/v3/stack.yml new file mode 100644 index 0000000..ad5c73d --- /dev/null +++ b/stacks/stack-orchestrator/stacks/v3/stack.yml @@ -0,0 +1,8 @@ +version: "1.0" +name: sushiswap-v3 +description: "SushiSwap v3 watcher stack" +repos: +containers: + - cerc/watcher-sushiswap-v3 +pods: + - watcher-sushiswap-v3 From e3a485f6b804b570b230f1e13cff397573a17a35 Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Thu, 13 Jun 2024 14:46:43 +0530 Subject: [PATCH 06/21] Exclude stacks folder in docker image --- .../container-build/cerc-watcher-v3/.dockerignore | 1 + .../container-build/cerc-watcher-v3/Dockerfile | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 stacks/stack-orchestrator/container-build/cerc-watcher-v3/.dockerignore create mode 100644 stacks/stack-orchestrator/container-build/cerc-watcher-v3/Dockerfile diff --git a/stacks/stack-orchestrator/container-build/cerc-watcher-v3/.dockerignore b/stacks/stack-orchestrator/container-build/cerc-watcher-v3/.dockerignore new file mode 100644 index 0000000..59da4af --- /dev/null +++ b/stacks/stack-orchestrator/container-build/cerc-watcher-v3/.dockerignore @@ -0,0 +1 @@ +stacks \ No newline at end of file diff --git a/stacks/stack-orchestrator/container-build/cerc-watcher-v3/Dockerfile b/stacks/stack-orchestrator/container-build/cerc-watcher-v3/Dockerfile new file mode 100644 index 0000000..9da5526 --- /dev/null +++ b/stacks/stack-orchestrator/container-build/cerc-watcher-v3/Dockerfile @@ -0,0 +1,11 @@ +# syntax=docker/dockerfile:1 +FROM node:18.17.1-alpine3.18 + +RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq + +WORKDIR /app + +COPY . . + +RUN echo "Installing dependencies and building sv3-watcher-ts" && \ + yarn && yarn build From b7bb44fa955ee287e2c22991d88ee1020c7d6542 Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Thu, 13 Jun 2024 15:02:23 +0530 Subject: [PATCH 07/21] Use sushiswap prefix for stack components --- ...> docker-compose-watcher-sushiswap-v3.yml} | 0 .../start-job-runner.sh | 0 .../start-server.sh | 0 .../watcher-v3/watcher-config-template.toml | 101 ------------------ .../.dockerignore | 0 .../cerc-watcher-sushiswap-v3/Dockerfile | 15 +++ .../cerc-watcher-v3/Dockerfile | 11 -- .../stacks/{v3 => sushiswap-v3}/stack.yml | 0 8 files changed, 15 insertions(+), 112 deletions(-) rename stacks/stack-orchestrator/compose/{docker-compose-watcher-v3.yml => docker-compose-watcher-sushiswap-v3.yml} (100%) rename stacks/stack-orchestrator/config/{watcher-v3 => watcher-sushiswap-v3}/start-job-runner.sh (100%) rename stacks/stack-orchestrator/config/{watcher-v3 => watcher-sushiswap-v3}/start-server.sh (100%) delete mode 100644 stacks/stack-orchestrator/config/watcher-v3/watcher-config-template.toml rename stacks/stack-orchestrator/container-build/{cerc-watcher-v3 => cerc-watcher-sushiswap-v3}/.dockerignore (100%) create mode 100644 stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/Dockerfile delete mode 100644 stacks/stack-orchestrator/container-build/cerc-watcher-v3/Dockerfile rename stacks/stack-orchestrator/stacks/{v3 => sushiswap-v3}/stack.yml (100%) diff --git a/stacks/stack-orchestrator/compose/docker-compose-watcher-v3.yml b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml similarity index 100% rename from stacks/stack-orchestrator/compose/docker-compose-watcher-v3.yml rename to stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml diff --git a/stacks/stack-orchestrator/config/watcher-v3/start-job-runner.sh b/stacks/stack-orchestrator/config/watcher-sushiswap-v3/start-job-runner.sh similarity index 100% rename from stacks/stack-orchestrator/config/watcher-v3/start-job-runner.sh rename to stacks/stack-orchestrator/config/watcher-sushiswap-v3/start-job-runner.sh diff --git a/stacks/stack-orchestrator/config/watcher-v3/start-server.sh b/stacks/stack-orchestrator/config/watcher-sushiswap-v3/start-server.sh similarity index 100% rename from stacks/stack-orchestrator/config/watcher-v3/start-server.sh rename to stacks/stack-orchestrator/config/watcher-sushiswap-v3/start-server.sh diff --git a/stacks/stack-orchestrator/config/watcher-v3/watcher-config-template.toml b/stacks/stack-orchestrator/config/watcher-v3/watcher-config-template.toml deleted file mode 100644 index bbc19c2..0000000 --- a/stacks/stack-orchestrator/config/watcher-v3/watcher-config-template.toml +++ /dev/null @@ -1,101 +0,0 @@ -[server] - host = "0.0.0.0" - port = 3008 - kind = "active" - gqlPath = "/" - - # Checkpointing state. - checkpointing = true - - # Checkpoint interval in number of blocks. - checkpointInterval = 2000 - - # Enable state creation - # CAUTION: Disable only if state creation is not desired or can be filled subsequently - enableState = false - - subgraphPath = "./subgraph-build" - - # Interval to restart wasm instance periodically - wasmRestartBlocksInterval = 20 - - # Interval in number of blocks at which to clear entities cache. - clearEntitiesCacheInterval = 1000 - - # Max block range for which to return events in eventsInRange GQL query. - # Use -1 for skipping check on block range. - maxEventsBlockRange = 1000 - - # Flag to specify whether RPC endpoint supports block hash as block tag parameter - rpcSupportsBlockHashParam = false - - # GQL cache settings - [server.gqlCache] - enabled = true - - # Max in-memory cache size (in bytes) (default 8 MB) - # maxCacheSize - - # GQL cache-control max-age settings (in seconds) - maxAge = 15 - timeTravelMaxAge = 86400 # 1 day - -[metrics] - host = "0.0.0.0" - port = 9000 - [metrics.gql] - port = 9001 - -[database] - type = "postgres" - host = "sushiswap-v3-watcher-db" - port = 5432 - database = "sushiswap-v3-watcher" - username = "vdbm" - password = "password" - synchronize = true - logging = false - -[upstream] - [upstream.ethServer] - rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS - - # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client) - rpcClient = true - - # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint - isFEVM = true - - # Boolean flag to filter event logs by contracts - filterLogsByAddresses = true - # Boolean flag to filter event logs by topics - filterLogsByTopics = true - - [upstream.cache] - name = "requests" - enabled = false - deleteOnStart = false - -[jobQueue] - dbConnectionString = "postgres://vdbm:password@sushiswap-v3-watcher-db/sushiswap-v3-watcher-job-queue" - maxCompletionLagInSecs = 300 - jobDelayInMilliSecs = 100 - eventsInBatch = 50 - subgraphEventsOrder = true - # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime - blockDelayInMilliSecs = 30000 - - # Boolean to switch between modes of processing events when starting the server. - # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. - # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head). - useBlockRanges = true - - # Block range in which logs are fetched during historical blocks processing - historicalLogsBlockRange = 2000 - - # Max block range of historical processing after which it waits for completion of events processing - # If set to -1 historical processing does not wait for events processing and completes till latest canonical block - historicalMaxFetchAhead = 10000 - - # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints - maxNewBlockRetries = 3 diff --git a/stacks/stack-orchestrator/container-build/cerc-watcher-v3/.dockerignore b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/.dockerignore similarity index 100% rename from stacks/stack-orchestrator/container-build/cerc-watcher-v3/.dockerignore rename to stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/.dockerignore diff --git a/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/Dockerfile b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/Dockerfile new file mode 100644 index 0000000..71aba48 --- /dev/null +++ b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/Dockerfile @@ -0,0 +1,15 @@ +FROM node:18.17.1-alpine3.18 + +RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq + +WORKDIR /app + +COPY . . + +# Get the latest Git commit hash and set in package.json +RUN COMMIT_HASH=$(git rev-parse HEAD) && \ + jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \ + mv tmp.json package.json + +RUN echo "Installing dependencies and building sv3-watcher-ts" && \ + yarn && yarn build diff --git a/stacks/stack-orchestrator/container-build/cerc-watcher-v3/Dockerfile b/stacks/stack-orchestrator/container-build/cerc-watcher-v3/Dockerfile deleted file mode 100644 index 9da5526..0000000 --- a/stacks/stack-orchestrator/container-build/cerc-watcher-v3/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# syntax=docker/dockerfile:1 -FROM node:18.17.1-alpine3.18 - -RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq - -WORKDIR /app - -COPY . . - -RUN echo "Installing dependencies and building sv3-watcher-ts" && \ - yarn && yarn build diff --git a/stacks/stack-orchestrator/stacks/v3/stack.yml b/stacks/stack-orchestrator/stacks/sushiswap-v3/stack.yml similarity index 100% rename from stacks/stack-orchestrator/stacks/v3/stack.yml rename to stacks/stack-orchestrator/stacks/sushiswap-v3/stack.yml From 15bb146e185878f9ded94521272ec0cb80ce6ad6 Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Thu, 13 Jun 2024 15:07:37 +0530 Subject: [PATCH 08/21] Modify config template for v3 watcher --- .../watcher-config-template.toml | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 stacks/stack-orchestrator/config/watcher-sushiswap-v3/watcher-config-template.toml diff --git a/stacks/stack-orchestrator/config/watcher-sushiswap-v3/watcher-config-template.toml b/stacks/stack-orchestrator/config/watcher-sushiswap-v3/watcher-config-template.toml new file mode 100644 index 0000000..817d532 --- /dev/null +++ b/stacks/stack-orchestrator/config/watcher-sushiswap-v3/watcher-config-template.toml @@ -0,0 +1,110 @@ +[server] + host = "0.0.0.0" + port = 3008 + kind = "active" + + # Checkpointing state. + checkpointing = true + + # Checkpoint interval in number of blocks. + checkpointInterval = 2000 + + # Enable state creation + # CAUTION: Disable only if state creation is not desired or can be filled subsequently + enableState = false + + subgraphPath = "./subgraph-build" + + # Interval to restart wasm instance periodically + wasmRestartBlocksInterval = 20 + + # Interval in number of blocks at which to clear entities cache. + clearEntitiesCacheInterval = 1000 + + # Flag to specify whether RPC endpoint supports block hash as block tag parameter + rpcSupportsBlockHashParam = false + + # Server GQL config + [server.gql] + path = "/" + + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 + + # Log directory for GQL requests + logDir = "./gql-logs" + + # GQL cache settings + [server.gql.cache] + enabled = true + + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize + + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day + +[metrics] + host = "0.0.0.0" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "sushiswap-v3-watcher-db" + port = 5432 + database = "sushiswap-v3-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + +[upstream] + [upstream.ethServer] + rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS + + # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client) + rpcClient = true + + # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint + isFEVM = true + + # Boolean flag to filter event logs by contracts + filterLogsByAddresses = true + # Boolean flag to filter event logs by topics + filterLogsByTopics = true + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@sushiswap-v3-watcher-db/sushiswap-v3-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 100 + eventsInBatch = 50 + subgraphEventsOrder = true + # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime + blockDelayInMilliSecs = 30000 + + # Number of blocks by which block processing lags behind head + blockProcessingOffset = 0 + + # Boolean to switch between modes of processing events when starting the server. + # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. + # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head). + useBlockRanges = true + + # Block range in which logs are fetched during historical blocks processing + historicalLogsBlockRange = 2000 + + # Max block range of historical processing after which it waits for completion of events processing + # If set to -1 historical processing does not wait for events processing and completes till latest canonical block + historicalMaxFetchAhead = 10000 + + # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints + maxNewBlockRetries = 3 From bbf6f0b2cc508376cc0a4e5c25b5e4025ff47773 Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Thu, 13 Jun 2024 15:24:23 +0530 Subject: [PATCH 09/21] Add container build script --- .../container-build/cerc-watcher-sushiswap-v3/build.sh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/build.sh diff --git a/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/build.sh b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/build.sh new file mode 100755 index 0000000..812b78e --- /dev/null +++ b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Build cerc/watcher-sushiswap-v3 + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/watcher-sushiswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/sushiswap-watcher-ts \ No newline at end of file From 72febfcf74957b5e3857f7cfa2218510fc15ccbb Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Thu, 13 Jun 2024 15:40:47 +0530 Subject: [PATCH 10/21] Add v3 watcher stack instructions --- .../stacks/sushiswap-v3/README.md | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 stacks/stack-orchestrator/stacks/sushiswap-v3/README.md diff --git a/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md b/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md new file mode 100644 index 0000000..db8ffe4 --- /dev/null +++ b/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md @@ -0,0 +1,103 @@ +# SushiSwap v3 Watcher + +## Clone the stack repo + +```bash +laconic-so --stack sushiswap-v3 setup-repositories --git-ssh --pull +``` + +## Build the container image + +```bash +laconic-so --stack sushiswap-v3 build-containers +``` +## Create a spec file for the deployment + +```bash +laconic-so --stack sushiswap-v3 deploy init --output sushiswap-v3-spec.yml +``` + +### Ports + +Edit `network` in the spec file to map container ports to host ports as required: + +```yml +... +network: + ports: + sushiswap-v3-watcher-db: + - '5432' + sushiswap-v3-watcher-job-runner: + - 9000:9000 + sushiswap-v3-watcher-server: + - 127.0.0.1:3008:3008 + - 9001:9001 +``` + +## Create a deployment from the spec file + +```bash +laconic-so --stack sushiswap-v3 deploy create --spec-file sushiswap-v3-spec.yml --deployment-dir sushiswap-v3-deployment +``` + +### Configuration + +Inside deployment directory, open the `config.env` file and set following env variables: + +```bash +# External Filecoin (ETH RPC) endpoint to point the watcher to +CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1 +``` + +## Start the deployment + +```bash +laconic-so deployment --dir sushiswap-v3-deployment start +``` + +* To list down and monitor the running containers: + + ```bash + # With status + docker ps -a + + # Check logs for a container + docker logs -f + ``` + +* Open the GQL playground at + + ```graphql + # Example query + query { + _meta { + block { + hash + number + timestamp + } + deployment + hasIndexingErrors + } + ``` + +## Clean up + +Stop all the sushiswap-v3 services running in background: + +```bash +# Only stop the docker containers +laconic-so deployment --dir sushiswap-v3-deployment stop + +# Run 'start' to restart the deployment +``` + +To stop all the sushiswap-v3 services and also delete data: + +```bash +# Stop the docker containers +laconic-so deployment --dir sushiswap-v3-deployment stop --delete-volumes + +# Remove deployment directory (deployment will have to be recreated for a re-run) +rm -r sushiswap-v3-deployment +``` \ No newline at end of file From 81752ca96139a9bc32fc9ba0eb2bc3f7f4bd525b Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Thu, 13 Jun 2024 15:48:09 +0530 Subject: [PATCH 11/21] Update stack repo URL in instructions --- stacks/stack-orchestrator/stacks/sushiswap-v3/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md b/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md index db8ffe4..5261062 100644 --- a/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md +++ b/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md @@ -3,7 +3,7 @@ ## Clone the stack repo ```bash -laconic-so --stack sushiswap-v3 setup-repositories --git-ssh --pull +laconic-so fetch-stack git.vdb.to/cerc-io/sushiswap-watcher-ts ``` ## Build the container image From 7dbaed75087fa4529effec12cd09ee3a8bcf152a Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Thu, 13 Jun 2024 16:43:08 +0530 Subject: [PATCH 12/21] Make volume names consistent --- .../compose/docker-compose-watcher-sushiswap-v3.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml index 102e784..efabb8b 100644 --- a/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml +++ b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml @@ -11,7 +11,7 @@ services: - POSTGRES_PASSWORD=password volumes: - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh - - sushiswap-v3_watcher_db_data:/var/lib/postgresql/data + - sushiswap_v3_watcher_db_data:/var/lib/postgresql/data ports: - "5432" healthcheck: @@ -60,10 +60,10 @@ services: volumes: - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh - - sushiswap-v3_watcher_gql_logs_data:/app/gql-logs + - sushiswap_v3_watcher_gql_logs_data:/app/gql-logs ports: - - "127.0.0.1:3008:3008" - - "9001:9001" + - "3008" + - "9001" healthcheck: test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"] interval: 20s @@ -74,5 +74,5 @@ services: - "host.docker.internal:host-gateway" volumes: - sushiswap-v3_watcher_db_data: - sushiswap-v3_watcher_gql_logs_data: + sushiswap_v3_watcher_db_data: + sushiswap_v3_watcher_gql_logs_data: From 9eba3ecc3db5c5c3fddf5713f5871019d1e87d27 Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Thu, 13 Jun 2024 16:48:15 +0530 Subject: [PATCH 13/21] Update sushiswap watcher image name --- .../compose/docker-compose-watcher-sushiswap-v3.yml | 4 ++-- .../container-build/cerc-watcher-sushiswap-v3/build.sh | 7 ------- .../.dockerignore | 0 .../Dockerfile | 2 +- .../container-build/cerc-watcher-sushiswap/build.sh | 7 +++++++ stacks/stack-orchestrator/stacks/sushiswap-v3/stack.yml | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) delete mode 100755 stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/build.sh rename stacks/stack-orchestrator/container-build/{cerc-watcher-sushiswap-v3 => cerc-watcher-sushiswap}/.dockerignore (100%) rename stacks/stack-orchestrator/container-build/{cerc-watcher-sushiswap-v3 => cerc-watcher-sushiswap}/Dockerfile (82%) create mode 100755 stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/build.sh diff --git a/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml index efabb8b..55595b6 100644 --- a/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml +++ b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml @@ -26,7 +26,7 @@ services: depends_on: sushiswap-v3-watcher-db: condition: service_healthy - image: cerc/watcher-sushiswap-v3:local + image: cerc/watcher-sushiswap:local environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} @@ -52,7 +52,7 @@ services: condition: service_healthy sushiswap-v3-watcher-job-runner: condition: service_healthy - image: cerc/watcher-sushiswap-v3:local + image: cerc/watcher-sushiswap:local environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} diff --git a/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/build.sh b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/build.sh deleted file mode 100755 index 812b78e..0000000 --- a/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/build.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -# Build cerc/watcher-sushiswap-v3 - -source ${CERC_CONTAINER_BASE_DIR}/build-base.sh -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -docker build -t cerc/watcher-sushiswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/sushiswap-watcher-ts \ No newline at end of file diff --git a/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/.dockerignore b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/.dockerignore similarity index 100% rename from stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/.dockerignore rename to stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/.dockerignore diff --git a/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/Dockerfile b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/Dockerfile similarity index 82% rename from stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/Dockerfile rename to stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/Dockerfile index 71aba48..62e580a 100644 --- a/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap-v3/Dockerfile +++ b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/Dockerfile @@ -11,5 +11,5 @@ RUN COMMIT_HASH=$(git rev-parse HEAD) && \ jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \ mv tmp.json package.json -RUN echo "Installing dependencies and building sv3-watcher-ts" && \ +RUN echo "Installing dependencies and building sushiswap-v3-watcher-ts" && \ yarn && yarn build diff --git a/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/build.sh b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/build.sh new file mode 100755 index 0000000..cff4cd9 --- /dev/null +++ b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Build cerc/watcher-sushiswap + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/watcher-sushiswap:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/sushiswap-watcher-ts \ No newline at end of file diff --git a/stacks/stack-orchestrator/stacks/sushiswap-v3/stack.yml b/stacks/stack-orchestrator/stacks/sushiswap-v3/stack.yml index ad5c73d..4b09c7a 100644 --- a/stacks/stack-orchestrator/stacks/sushiswap-v3/stack.yml +++ b/stacks/stack-orchestrator/stacks/sushiswap-v3/stack.yml @@ -3,6 +3,6 @@ name: sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: containers: - - cerc/watcher-sushiswap-v3 + - cerc/watcher-sushiswap pods: - watcher-sushiswap-v3 From 8451ac1805f8518b370343787f1e99a87e259bc2 Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Thu, 13 Jun 2024 18:36:59 +0530 Subject: [PATCH 14/21] Update port mapping and stack path in readme --- stacks/stack-orchestrator/stacks/sushiswap-v3/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md b/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md index 5261062..c2a4abb 100644 --- a/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md +++ b/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md @@ -3,7 +3,7 @@ ## Clone the stack repo ```bash -laconic-so fetch-stack git.vdb.to/cerc-io/sushiswap-watcher-ts +laconic-so fetch-stack ~/cerc/sushiswap-watcher-ts/stack-orchestrator/stacks/sushiswap-v3 ``` ## Build the container image @@ -11,6 +11,7 @@ laconic-so fetch-stack git.vdb.to/cerc-io/sushiswap-watcher-ts ```bash laconic-so --stack sushiswap-v3 build-containers ``` + ## Create a spec file for the deployment ```bash @@ -30,7 +31,7 @@ network: sushiswap-v3-watcher-job-runner: - 9000:9000 sushiswap-v3-watcher-server: - - 127.0.0.1:3008:3008 + - 3008:3008 - 9001:9001 ``` From 7adee92f9aa963676260cf429b14a9a9b6a5a590 Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Thu, 13 Jun 2024 18:53:22 +0530 Subject: [PATCH 15/21] Update mount paths for mounted files --- .../compose/docker-compose-watcher-sushiswap-v3.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml index 55595b6..2139aba 100644 --- a/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml +++ b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml @@ -33,7 +33,7 @@ services: command: ["bash", "./start-job-runner.sh"] volumes: - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml - - ../config/watcher-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh + - ../config/watcher-sushiswap-v3/start-job-runner.sh:/app/packages/v3-watcher/start-job-runner.sh ports: - "9000:9000" healthcheck: @@ -59,7 +59,7 @@ services: command: ["bash", "./start-server.sh"] volumes: - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml - - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh + - ../config/watcher-sushiswap-v3/start-server.sh:/app/packages/v3-watcher/start-server.sh - sushiswap_v3_watcher_gql_logs_data:/app/gql-logs ports: - "3008" From b4bb9830bea4058156233a3ccaf9783293aef01b Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Thu, 13 Jun 2024 19:02:26 +0530 Subject: [PATCH 16/21] Set working_dir to v3 watcher in compose file --- .../compose/docker-compose-watcher-sushiswap-v3.yml | 2 ++ stacks/stack-orchestrator/stacks/sushiswap-v3/README.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml index 2139aba..0bcece5 100644 --- a/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml +++ b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml @@ -30,6 +30,7 @@ services: environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + working_dir: /app/packages/v3-watcher command: ["bash", "./start-job-runner.sh"] volumes: - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml @@ -56,6 +57,7 @@ services: environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + working_dir: /app/packages/v3-watcher command: ["bash", "./start-server.sh"] volumes: - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml diff --git a/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md b/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md index c2a4abb..a21eaaa 100644 --- a/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md +++ b/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md @@ -27,7 +27,7 @@ Edit `network` in the spec file to map container ports to host ports as required network: ports: sushiswap-v3-watcher-db: - - '5432' + - 15432:5432 sushiswap-v3-watcher-job-runner: - 9000:9000 sushiswap-v3-watcher-server: From 28bfcfffc35948dc050347afaaa36f6f778253ae Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Fri, 14 Jun 2024 10:47:14 +0530 Subject: [PATCH 17/21] Fix mount paths for mounted files --- .../compose/docker-compose-watcher-sushiswap-v3.yml | 6 +++--- .../watcher-sushiswap-v3/watcher-config-template.toml | 2 +- .../cerc-watcher-sushiswap/.dockerignore | 2 +- .../container-build/cerc-watcher-sushiswap/Dockerfile | 2 +- .../container-build/cerc-watcher-sushiswap/build.sh | 2 +- .../stack-orchestrator/stacks/sushiswap-v3/README.md | 10 +++++----- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml index 0bcece5..b1f99bf 100644 --- a/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml +++ b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml @@ -33,7 +33,7 @@ services: working_dir: /app/packages/v3-watcher command: ["bash", "./start-job-runner.sh"] volumes: - - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/packages/v3-watcher/environments/watcher-config-template.toml - ../config/watcher-sushiswap-v3/start-job-runner.sh:/app/packages/v3-watcher/start-job-runner.sh ports: - "9000:9000" @@ -60,9 +60,9 @@ services: working_dir: /app/packages/v3-watcher command: ["bash", "./start-server.sh"] volumes: - - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/packages/v3-watcher/environments/watcher-config-template.toml - ../config/watcher-sushiswap-v3/start-server.sh:/app/packages/v3-watcher/start-server.sh - - sushiswap_v3_watcher_gql_logs_data:/app/gql-logs + - sushiswap_v3_watcher_gql_logs_data:/app/packages/v3-watcher/gql-logs ports: - "3008" - "9001" diff --git a/stacks/stack-orchestrator/config/watcher-sushiswap-v3/watcher-config-template.toml b/stacks/stack-orchestrator/config/watcher-sushiswap-v3/watcher-config-template.toml index 817d532..a69d858 100644 --- a/stacks/stack-orchestrator/config/watcher-sushiswap-v3/watcher-config-template.toml +++ b/stacks/stack-orchestrator/config/watcher-sushiswap-v3/watcher-config-template.toml @@ -26,7 +26,7 @@ # Server GQL config [server.gql] - path = "/" + path = "/graphql" # Max block range for which to return events in eventsInRange GQL query. # Use -1 for skipping check on block range. diff --git a/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/.dockerignore b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/.dockerignore index 59da4af..6013216 100644 --- a/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/.dockerignore +++ b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/.dockerignore @@ -1 +1 @@ -stacks \ No newline at end of file +stacks diff --git a/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/Dockerfile b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/Dockerfile index 62e580a..6c16543 100644 --- a/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/Dockerfile +++ b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/Dockerfile @@ -11,5 +11,5 @@ RUN COMMIT_HASH=$(git rev-parse HEAD) && \ jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \ mv tmp.json package.json -RUN echo "Installing dependencies and building sushiswap-v3-watcher-ts" && \ +RUN echo "Installing dependencies and building sushiswap-watcher-ts" && \ yarn && yarn build diff --git a/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/build.sh b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/build.sh index cff4cd9..2bdc814 100755 --- a/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/build.sh +++ b/stacks/stack-orchestrator/container-build/cerc-watcher-sushiswap/build.sh @@ -4,4 +4,4 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -docker build -t cerc/watcher-sushiswap:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/sushiswap-watcher-ts \ No newline at end of file +docker build -t cerc/watcher-sushiswap:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/sushiswap-watcher-ts diff --git a/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md b/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md index a21eaaa..e27e95f 100644 --- a/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md +++ b/stacks/stack-orchestrator/stacks/sushiswap-v3/README.md @@ -3,19 +3,19 @@ ## Clone the stack repo ```bash -laconic-so fetch-stack ~/cerc/sushiswap-watcher-ts/stack-orchestrator/stacks/sushiswap-v3 +laconic-so fetch-stack git.vdb.to/cerc-io/sushiswap-watcher-ts ``` ## Build the container image ```bash -laconic-so --stack sushiswap-v3 build-containers +laconic-so --stack ~/cerc/sushiswap-watcher-ts/stacks/stack-orchestrator/stacks/sushiswap-v3 build-containers ``` ## Create a spec file for the deployment ```bash -laconic-so --stack sushiswap-v3 deploy init --output sushiswap-v3-spec.yml +laconic-so --stack ~/cerc/sushiswap-watcher-ts/stacks/stack-orchestrator/stacks/sushiswap-v3 deploy init --output sushiswap-v3-spec.yml ``` ### Ports @@ -38,7 +38,7 @@ network: ## Create a deployment from the spec file ```bash -laconic-so --stack sushiswap-v3 deploy create --spec-file sushiswap-v3-spec.yml --deployment-dir sushiswap-v3-deployment +laconic-so --stack ~/cerc/sushiswap-watcher-ts/stacks/stack-orchestrator/stacks/sushiswap-v3 deploy create --spec-file sushiswap-v3-spec.yml --deployment-dir sushiswap-v3-deployment ``` ### Configuration @@ -101,4 +101,4 @@ laconic-so deployment --dir sushiswap-v3-deployment stop --delete-volumes # Remove deployment directory (deployment will have to be recreated for a re-run) rm -r sushiswap-v3-deployment -``` \ No newline at end of file +``` From 1eab60447e8d5262018c1580ef5bce03165d68b3 Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Fri, 14 Jun 2024 11:00:59 +0530 Subject: [PATCH 18/21] Add docker compose file for v2 --- .../docker-compose-watcher-sushiswap-v2.yml | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v2.yml diff --git a/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v2.yml b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v2.yml new file mode 100644 index 0000000..05122e2 --- /dev/null +++ b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v2.yml @@ -0,0 +1,80 @@ +version: '3.2' + +services: + sushiswap-v2-watcher-db: + restart: unless-stopped + image: postgres:14-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_MULTIPLE_DATABASES=sushiswap-v2-watcher,sushiswap-v2-watcher-job-queue + - POSTGRES_EXTENSION=sushiswap-v2-watcher-job-queue:pgcrypto + - POSTGRES_PASSWORD=password + volumes: + - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh + - sushiswap_v2_watcher_db_data:/var/lib/postgresql/data + ports: + - "5432" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + + sushiswap-v2-watcher-job-runner: + restart: unless-stopped + depends_on: + sushiswap-v2-watcher-db: + condition: service_healthy + image: cerc/watcher-sushiswap:local + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + working_dir: /app/packages/v2-watcher + command: ["bash", "./start-job-runner.sh"] + volumes: + - ../config/watcher-sushiswap-v2/watcher-config-template.toml:/app/packages/v2-watcher/environments/watcher-config-template.toml + - ../config/watcher-sushiswap-v2/start-job-runner.sh:/app/packages/v2-watcher/start-job-runner.sh + ports: + - "9000:9000" + healthcheck: + test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + sushiswap-v2-watcher-server: + restart: unless-stopped + depends_on: + sushiswap-v2-watcher-db: + condition: service_healthy + sushiswap-v2-watcher-job-runner: + condition: service_healthy + image: cerc/watcher-sushiswap:local + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS} + working_dir: /app/packages/v2-watcher + command: ["bash", "./start-server.sh"] + volumes: + - ../config/watcher-sushiswap-v2/watcher-config-template.toml:/app/packages/v2-watcher/environments/watcher-config-template.toml + - ../config/watcher-sushiswap-v2/start-server.sh:/app/packages/v2-watcher/start-server.sh + - sushiswap_v2_watcher_gql_logs_data:/app/packages/v2-watcher/gql-logs + ports: + - "3008" + - "9001" + healthcheck: + test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + sushiswap_v2_watcher_db_data: + sushiswap_v2_watcher_gql_logs_data: From fb8dfd0e518d8cb25981e227ff58d7d9d259abc3 Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Fri, 14 Jun 2024 11:06:43 +0530 Subject: [PATCH 19/21] Add config template and scripts for v3 watcher --- .../watcher-sushiswap-v2/start-job-runner.sh | 24 ++++ .../watcher-sushiswap-v2/start-server.sh | 24 ++++ .../watcher-config-template.toml | 110 ++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100755 stacks/stack-orchestrator/config/watcher-sushiswap-v2/start-job-runner.sh create mode 100755 stacks/stack-orchestrator/config/watcher-sushiswap-v2/start-server.sh create mode 100644 stacks/stack-orchestrator/config/watcher-sushiswap-v2/watcher-config-template.toml diff --git a/stacks/stack-orchestrator/config/watcher-sushiswap-v2/start-job-runner.sh b/stacks/stack-orchestrator/config/watcher-sushiswap-v2/start-job-runner.sh new file mode 100755 index 0000000..7a7a83b --- /dev/null +++ b/stacks/stack-orchestrator/config/watcher-sushiswap-v2/start-job-runner.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +set -u + +echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) + +# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array +RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') + +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +echo "Running job-runner..." +DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js diff --git a/stacks/stack-orchestrator/config/watcher-sushiswap-v2/start-server.sh b/stacks/stack-orchestrator/config/watcher-sushiswap-v2/start-server.sh new file mode 100755 index 0000000..9aaa77e --- /dev/null +++ b/stacks/stack-orchestrator/config/watcher-sushiswap-v2/start-server.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +set -u + +echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) + +# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array +RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') + +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +echo "Running server..." +DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js diff --git a/stacks/stack-orchestrator/config/watcher-sushiswap-v2/watcher-config-template.toml b/stacks/stack-orchestrator/config/watcher-sushiswap-v2/watcher-config-template.toml new file mode 100644 index 0000000..fec02a9 --- /dev/null +++ b/stacks/stack-orchestrator/config/watcher-sushiswap-v2/watcher-config-template.toml @@ -0,0 +1,110 @@ +[server] + host = "0.0.0.0" + port = 3008 + kind = "active" + + # Checkpointing state. + checkpointing = true + + # Checkpoint interval in number of blocks. + checkpointInterval = 2000 + + # Enable state creation + # CAUTION: Disable only if state creation is not desired or can be filled subsequently + enableState = false + + subgraphPath = "./subgraph-build" + + # Interval to restart wasm instance periodically + wasmRestartBlocksInterval = 20 + + # Interval in number of blocks at which to clear entities cache. + clearEntitiesCacheInterval = 1000 + + # Flag to specify whether RPC endpoint supports block hash as block tag parameter + rpcSupportsBlockHashParam = false + + # Server GQL config + [server.gql] + path = "/graphql" + + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 + + # Log directory for GQL requests + logDir = "./gql-logs" + + # GQL cache settings + [server.gql.cache] + enabled = true + + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize + + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day + +[metrics] + host = "0.0.0.0" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "sushiswap-v2-watcher-db" + port = 5432 + database = "sushiswap-v2-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + +[upstream] + [upstream.ethServer] + rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS + + # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client) + rpcClient = true + + # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint + isFEVM = true + + # Boolean flag to filter event logs by contracts + filterLogsByAddresses = true + # Boolean flag to filter event logs by topics + filterLogsByTopics = true + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@sushiswap-v2-watcher-db/sushiswap-v2-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 100 + eventsInBatch = 50 + subgraphEventsOrder = true + # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime + blockDelayInMilliSecs = 30000 + + # Number of blocks by which block processing lags behind head + blockProcessingOffset = 0 + + # Boolean to switch between modes of processing events when starting the server. + # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. + # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head). + useBlockRanges = true + + # Block range in which logs are fetched during historical blocks processing + historicalLogsBlockRange = 2000 + + # Max block range of historical processing after which it waits for completion of events processing + # If set to -1 historical processing does not wait for events processing and completes till latest canonical block + historicalMaxFetchAhead = 10000 + + # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints + maxNewBlockRetries = 3 From 31c39ec8dabe372a32a4a20803b35bf6f4d9d45f Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Fri, 14 Jun 2024 11:13:32 +0530 Subject: [PATCH 20/21] Add v2 watcher stack instructions --- .../stacks/sushiswap-v2/README.md | 104 ++++++++++++++++++ .../stacks/sushiswap-v2/stack.yml | 8 ++ 2 files changed, 112 insertions(+) create mode 100644 stacks/stack-orchestrator/stacks/sushiswap-v2/README.md create mode 100644 stacks/stack-orchestrator/stacks/sushiswap-v2/stack.yml diff --git a/stacks/stack-orchestrator/stacks/sushiswap-v2/README.md b/stacks/stack-orchestrator/stacks/sushiswap-v2/README.md new file mode 100644 index 0000000..93c58e1 --- /dev/null +++ b/stacks/stack-orchestrator/stacks/sushiswap-v2/README.md @@ -0,0 +1,104 @@ +# SushiSwap v2 Watcher + +## Clone the stack repo + +```bash +laconic-so fetch-stack git.vdb.to/deep-stack/sushiswap-watcher-ts +``` + +## Build the container image + +```bash +laconic-so --stack ~/cerc/sushiswap-watcher-ts/stacks/stack-orchestrator/stacks/sushiswap-v2 build-containers +``` + +## Create a spec file for the deployment + +```bash +laconic-so --stack ~/cerc/sushiswap-watcher-ts/stacks/stack-orchestrator/stacks/sushiswap-v2 deploy init --output sushiswap-v2-spec.yml +``` + +### Ports + +Edit `network` in the spec file to map container ports to host ports as required: + +```yml +... +network: + ports: + sushiswap-v2-watcher-db: + - 15432:5432 + sushiswap-v2-watcher-job-runner: + - 9000:9000 + sushiswap-v2-watcher-server: + - 3008:3008 + - 9001:9001 +``` + +## Create a deployment from the spec file + +```bash +laconic-so --stack ~/cerc/sushiswap-watcher-ts/stacks/stack-orchestrator/stacks/sushiswap-v2 deploy create --spec-file sushiswap-v2-spec.yml --deployment-dir sushiswap-v2-deployment +``` + +### Configuration + +Inside deployment directory, open the `config.env` file and set following env variables: + +```bash +# External Filecoin (ETH RPC) endpoint to point the watcher to +CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1 +``` + +## Start the deployment + +```bash +laconic-so deployment --dir sushiswap-v2-deployment start +``` + +* To list down and monitor the running containers: + + ```bash + # With status + docker ps -a + + # Check logs for a container + docker logs -f + ``` + +* Open the GQL playground at + + ```graphql + # Example query + query { + _meta { + block { + hash + number + timestamp + } + deployment + hasIndexingErrors + } + ``` + +## Clean up + +Stop all the sushiswap-v2 services running in background: + +```bash +# Only stop the docker containers +laconic-so deployment --dir sushiswap-v2-deployment stop + +# Run 'start' to restart the deployment +``` + +To stop all the sushiswap-v2 services and also delete data: + +```bash +# Stop the docker containers +laconic-so deployment --dir sushiswap-v2-deployment stop --delete-volumes + +# Remove deployment directory (deployment will have to be recreated for a re-run) +rm -r sushiswap-v2-deployment +``` diff --git a/stacks/stack-orchestrator/stacks/sushiswap-v2/stack.yml b/stacks/stack-orchestrator/stacks/sushiswap-v2/stack.yml new file mode 100644 index 0000000..8ed7246 --- /dev/null +++ b/stacks/stack-orchestrator/stacks/sushiswap-v2/stack.yml @@ -0,0 +1,8 @@ +version: "1.0" +name: sushiswap-v2 +description: "SushiSwap v2 watcher stack" +repos: +containers: + - cerc/watcher-sushiswap +pods: + - watcher-sushiswap-v2 From b3bb0be4bde2e8946a5b524a03fce01f027d717c Mon Sep 17 00:00:00 2001 From: IshaVenikar Date: Fri, 14 Jun 2024 11:57:24 +0530 Subject: [PATCH 21/21] Add job runner and server scripts in common folder --- .../docker-compose-watcher-sushiswap-v2.yml | 6 ++--- .../docker-compose-watcher-sushiswap-v3.yml | 6 ++--- .../start-job-runner.sh | 0 .../start-server.sh | 0 .../watcher-sushiswap-v3/start-job-runner.sh | 24 ------------------- .../watcher-sushiswap-v3/start-server.sh | 24 ------------------- .../stacks/sushiswap-v2/README.md | 2 +- 7 files changed, 7 insertions(+), 55 deletions(-) rename stacks/stack-orchestrator/config/{watcher-sushiswap-v2 => common}/start-job-runner.sh (100%) rename stacks/stack-orchestrator/config/{watcher-sushiswap-v2 => common}/start-server.sh (100%) delete mode 100755 stacks/stack-orchestrator/config/watcher-sushiswap-v3/start-job-runner.sh delete mode 100755 stacks/stack-orchestrator/config/watcher-sushiswap-v3/start-server.sh diff --git a/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v2.yml b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v2.yml index 05122e2..2762498 100644 --- a/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v2.yml +++ b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v2.yml @@ -34,9 +34,9 @@ services: command: ["bash", "./start-job-runner.sh"] volumes: - ../config/watcher-sushiswap-v2/watcher-config-template.toml:/app/packages/v2-watcher/environments/watcher-config-template.toml - - ../config/watcher-sushiswap-v2/start-job-runner.sh:/app/packages/v2-watcher/start-job-runner.sh + - ../config/common/start-job-runner.sh:/app/packages/v2-watcher/start-job-runner.sh ports: - - "9000:9000" + - "9000" healthcheck: test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"] interval: 20s @@ -61,7 +61,7 @@ services: command: ["bash", "./start-server.sh"] volumes: - ../config/watcher-sushiswap-v2/watcher-config-template.toml:/app/packages/v2-watcher/environments/watcher-config-template.toml - - ../config/watcher-sushiswap-v2/start-server.sh:/app/packages/v2-watcher/start-server.sh + - ../config/common/start-server.sh:/app/packages/v2-watcher/start-server.sh - sushiswap_v2_watcher_gql_logs_data:/app/packages/v2-watcher/gql-logs ports: - "3008" diff --git a/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml index b1f99bf..a36cf94 100644 --- a/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml +++ b/stacks/stack-orchestrator/compose/docker-compose-watcher-sushiswap-v3.yml @@ -34,9 +34,9 @@ services: command: ["bash", "./start-job-runner.sh"] volumes: - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/packages/v3-watcher/environments/watcher-config-template.toml - - ../config/watcher-sushiswap-v3/start-job-runner.sh:/app/packages/v3-watcher/start-job-runner.sh + - ../config/common/start-job-runner.sh:/app/packages/v3-watcher/start-job-runner.sh ports: - - "9000:9000" + - "9000" healthcheck: test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"] interval: 20s @@ -61,7 +61,7 @@ services: command: ["bash", "./start-server.sh"] volumes: - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/packages/v3-watcher/environments/watcher-config-template.toml - - ../config/watcher-sushiswap-v3/start-server.sh:/app/packages/v3-watcher/start-server.sh + - ../config/common/start-server.sh:/app/packages/v3-watcher/start-server.sh - sushiswap_v3_watcher_gql_logs_data:/app/packages/v3-watcher/gql-logs ports: - "3008" diff --git a/stacks/stack-orchestrator/config/watcher-sushiswap-v2/start-job-runner.sh b/stacks/stack-orchestrator/config/common/start-job-runner.sh similarity index 100% rename from stacks/stack-orchestrator/config/watcher-sushiswap-v2/start-job-runner.sh rename to stacks/stack-orchestrator/config/common/start-job-runner.sh diff --git a/stacks/stack-orchestrator/config/watcher-sushiswap-v2/start-server.sh b/stacks/stack-orchestrator/config/common/start-server.sh similarity index 100% rename from stacks/stack-orchestrator/config/watcher-sushiswap-v2/start-server.sh rename to stacks/stack-orchestrator/config/common/start-server.sh diff --git a/stacks/stack-orchestrator/config/watcher-sushiswap-v3/start-job-runner.sh b/stacks/stack-orchestrator/config/watcher-sushiswap-v3/start-job-runner.sh deleted file mode 100755 index 7a7a83b..0000000 --- a/stacks/stack-orchestrator/config/watcher-sushiswap-v3/start-job-runner.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh - -set -e -if [ -n "$CERC_SCRIPT_DEBUG" ]; then - set -x -fi -set -u - -echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" - -# Read in the config template TOML file and modify it -WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) - -# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array -RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') - -WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ - sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") - -# Write the modified content to a new file -echo "$WATCHER_CONFIG" > environments/local.toml - -echo "Running job-runner..." -DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js diff --git a/stacks/stack-orchestrator/config/watcher-sushiswap-v3/start-server.sh b/stacks/stack-orchestrator/config/watcher-sushiswap-v3/start-server.sh deleted file mode 100755 index 9aaa77e..0000000 --- a/stacks/stack-orchestrator/config/watcher-sushiswap-v3/start-server.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh - -set -e -if [ -n "$CERC_SCRIPT_DEBUG" ]; then - set -x -fi -set -u - -echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}" - -# Read in the config template TOML file and modify it -WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) - -# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array -RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/') - -WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ - sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ") - -# Write the modified content to a new file -echo "$WATCHER_CONFIG" > environments/local.toml - -echo "Running server..." -DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js diff --git a/stacks/stack-orchestrator/stacks/sushiswap-v2/README.md b/stacks/stack-orchestrator/stacks/sushiswap-v2/README.md index 93c58e1..c8acfdc 100644 --- a/stacks/stack-orchestrator/stacks/sushiswap-v2/README.md +++ b/stacks/stack-orchestrator/stacks/sushiswap-v2/README.md @@ -3,7 +3,7 @@ ## Clone the stack repo ```bash -laconic-so fetch-stack git.vdb.to/deep-stack/sushiswap-watcher-ts +laconic-so fetch-stack git.vdb.to/cerc-io/sushiswap-watcher-ts ``` ## Build the container image