From a04730e7acc6d2055887285bef16853f7b449aae Mon Sep 17 00:00:00 2001 From: Nabarun Gogoi Date: Mon, 13 Nov 2023 11:13:55 +0530 Subject: [PATCH] Add a merkl-sushiswap-v3 watcher stack (#641) * Add a merkl-sushiswap-v3 watcher stack * Remove unrequired image from list --- ...ker-compose-watcher-merkl-sushiswap-v3.yml | 77 +++++++++++++++ .../start-job-runner.sh | 20 ++++ .../start-server.sh | 23 +++++ .../watcher-config-template.toml | 98 +++++++++++++++++++ .../Dockerfile | 10 ++ .../cerc-watcher-merkl-sushiswap-v3/build.sh | 11 +++ .../data/container-image-list.txt | 1 + stack_orchestrator/data/pod-list.txt | 1 + stack_orchestrator/data/repository-list.txt | 1 + .../data/stacks/graph-node/README.md | 2 +- .../data/stacks/merkl-sushiswap-v3/README.md | 81 +++++++++++++++ .../data/stacks/merkl-sushiswap-v3/stack.yml | 9 ++ 12 files changed, 333 insertions(+), 1 deletion(-) create mode 100644 stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml create mode 100755 stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh create mode 100755 stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh create mode 100644 stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml create mode 100644 stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile create mode 100755 stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/build.sh create mode 100644 stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md create mode 100644 stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml new file mode 100644 index 00000000..62e848e8 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml @@ -0,0 +1,77 @@ +version: '3.2' + +services: + merkl-sushiswap-v3-watcher-db: + restart: unless-stopped + image: postgres:14-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_MULTIPLE_DATABASES=merkl-sushiswap-v3-watcher,merkl-sushiswap-v3-watcher-job-queue + - POSTGRES_EXTENSION=merkl-sushiswap-v3-watcher-job-queue:pgcrypto + - POSTGRES_PASSWORD=password + volumes: + - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh + - merkl_sushiswap_v3_watcher_db_data:/var/lib/postgresql/data + ports: + - "127.0.0.1:15432:5432" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + + merkl-sushiswap-v3-watcher-job-runner: + restart: unless-stopped + depends_on: + merkl-sushiswap-v3-watcher-db: + condition: service_healthy + image: cerc/watcher-merkl-sushiswap-v3:local + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} + command: ["bash", "./start-job-runner.sh"] + volumes: + - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-merkl-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh + ports: + - "127.0.0.1:9000:9000" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "9000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + merkl-sushiswap-v3-watcher-server: + restart: unless-stopped + depends_on: + merkl-sushiswap-v3-watcher-db: + condition: service_healthy + merkl-sushiswap-v3-watcher-job-runner: + condition: service_healthy + image: cerc/watcher-merkl-sushiswap-v3:local + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} + SUSHISWAP_START_BLOCK: ${SUSHISWAP_START_BLOCK:- 2867560} + command: ["bash", "./start-server.sh"] + volumes: + - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh + ports: + - "127.0.0.1:3007:3008" + - "127.0.0.1:9001:9001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3008"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + merkl_sushiswap_v3_watcher_db_data: diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh new file mode 100755 index 00000000..819b1096 --- /dev/null +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +set -u + +echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +echo "Running job-runner..." +DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh new file mode 100755 index 00000000..1b14f2e3 --- /dev/null +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +set -u + +echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +echo "Initializing watcher..." +yarn fill --start-block $SUSHISWAP_START_BLOCK --end-block $((SUSHISWAP_START_BLOCK + 1)) + +echo "Running server..." +DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml new file mode 100644 index 00000000..053e5544 --- /dev/null +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml @@ -0,0 +1,98 @@ +[server] + host = "0.0.0.0" + port = 3008 + kind = "active" + + # Checkpointing state. + checkpointing = true + + # Checkpoint interval in number of blocks. + checkpointInterval = 2000 + + # Enable state creation + # CAUTION: Disable only if state creation is not desired or can be filled subsequently + enableState = false + + subgraphPath = "./subgraph-build" + + # Interval to restart wasm instance periodically + wasmRestartBlocksInterval = 20 + + # Interval in number of blocks at which to clear entities cache. + clearEntitiesCacheInterval = 1000 + + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 + + # Flag to specify whether RPC endpoint supports block hash as block tag parameter + rpcSupportsBlockHashParam = false + + # GQL cache settings + [server.gqlCache] + enabled = true + + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize + + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day + +[metrics] + host = "127.0.0.1" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "merkl-sushiswap-v3-watcher-db" + port = 5432 + database = "merkl-sushiswap-v3-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + +[upstream] + [upstream.ethServer] + rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT" + + # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client) + rpcClient = true + + # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint + isFEVM = true + + # Boolean flag to filter event logs by contracts + filterLogsByAddresses = true + # Boolean flag to filter event logs by topics + filterLogsByTopics = false + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@merkl-sushiswap-v3-watcher-db/merkl-sushiswap-v3-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 100 + eventsInBatch = 50 + subgraphEventsOrder = true + blockDelayInMilliSecs = 2000 + prefetchBlocksInMem = false + prefetchBlockCount = 10 + + # Boolean to switch between modes of processing events when starting the server. + # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. + # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head). + useBlockRanges = true + + # Block range in which logs are fetched during historical blocks processing + historicalLogsBlockRange = 2000 + + # Max block range of historical processing after which it waits for completion of events processing + # If set to -1 historical processing does not wait for events processing and completes till latest canonical block + historicalMaxFetchAhead = 10000 diff --git a/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile new file mode 100644 index 00000000..e09738ac --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile @@ -0,0 +1,10 @@ +FROM node:18.17.1-alpine3.18 + +RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq + +WORKDIR /app + +COPY . . + +RUN echo "Installing dependencies and building merkl-sushiswap-v3-watcher-ts" && \ + yarn && yarn build diff --git a/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/build.sh b/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/build.sh new file mode 100755 index 00000000..b53ee621 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/build.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Build cerc/watcher-merkl-sushiswap-v3 + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/my-new-stack:local -f ${CERC_REPO_BASE_DIR}/my-new-stack/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/my-new-stack + +docker build -t cerc/watcher-merkl-sushiswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/merkl-sushiswap-v3-watcher-ts diff --git a/stack_orchestrator/data/container-image-list.txt b/stack_orchestrator/data/container-image-list.txt index 256f0a6f..36df0201 100644 --- a/stack_orchestrator/data/container-image-list.txt +++ b/stack_orchestrator/data/container-image-list.txt @@ -57,3 +57,4 @@ cerc/nitro-contracts cerc/mobymask-snap cerc/ponder cerc/nitro-rpc-client +cerc/watcher-merkl-sushiswap-v3 \ No newline at end of file diff --git a/stack_orchestrator/data/pod-list.txt b/stack_orchestrator/data/pod-list.txt index 4ba1bac0..7b91600d 100644 --- a/stack_orchestrator/data/pod-list.txt +++ b/stack_orchestrator/data/pod-list.txt @@ -43,3 +43,4 @@ nitro-contracts mobymask-snap ponder ipld-eth-server-payments +merkl-sushiswap-v3 diff --git a/stack_orchestrator/data/repository-list.txt b/stack_orchestrator/data/repository-list.txt index ceaa910c..f6696464 100644 --- a/stack_orchestrator/data/repository-list.txt +++ b/stack_orchestrator/data/repository-list.txt @@ -47,3 +47,4 @@ github.com/cerc-io/go-nitro github.com/cerc-io/ts-nitro github.com/cerc-io/mobymask-snap github.com/cerc-io/ponder +github.com/cerc-io/merkl-sushiswap-v3-watcher-ts diff --git a/stack_orchestrator/data/stacks/graph-node/README.md b/stack_orchestrator/data/stacks/graph-node/README.md index 0527efc0..df3ae1eb 100644 --- a/stack_orchestrator/data/stacks/graph-node/README.md +++ b/stack_orchestrator/data/stacks/graph-node/README.md @@ -59,7 +59,7 @@ ports: Create deployment: ```bash -laconic-so deploy create --spec-file graph-node-spec.yml --deployment-dir graph-node-deployment +laconic-so --stack graph-node deploy create --spec-file graph-node-spec.yml --deployment-dir graph-node-deployment ``` ## Start the stack diff --git a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md new file mode 100644 index 00000000..4284c2ad --- /dev/null +++ b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md @@ -0,0 +1,81 @@ +# Merkl SushiSwap v3 Watcher + +## Setup + +Clone required repositories: + +```bash +laconic-so --stack merkl-sushiswap-v3 setup-repositories --git-ssh --pull +``` + +Build the container images: + +```bash +laconic-so --stack merkl-sushiswap-v3 build-containers +``` + +## Deploy + +### Configuration + +Create and update an env file to be used in the next step: + + ```bash + # External Filecoin (ETH RPC) endpoint to point the watcher + CERC_ETH_RPC_ENDPOINT= + ``` + +### Deploy the stack + +```bash +laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 --env-file up +``` + +* To list down and monitor the running containers: + + ```bash + laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 ps + + # With status + docker ps -a + + # Check logs for a container + docker logs -f + ``` + +* Open the GQL playground at http://localhost:3007/graphql + + ```graphql + { + _meta { + block { + number + timestamp + } + hasIndexingErrors + } + + factories { + id + poolCount + } + } + ``` + +## Clean up + +Stop all the services running in background: + +```bash +laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 down +``` + +Clear volumes created by this stack: + +```bash +# List all relevant volumes +docker volume ls -q --filter "name=merkl_sushiswap_v3" + +# Remove all the listed volumes +docker volume rm $(docker volume ls -q --filter "name=merkl_sushiswap_v3") +``` diff --git a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml new file mode 100644 index 00000000..e3b5abf1 --- /dev/null +++ b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml @@ -0,0 +1,9 @@ +version: "1.0" +name: merkl-sushiswap-v3 +description: "SushiSwap v3 watcher stack" +repos: + - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.0 +containers: + - cerc/watcher-merkl-sushiswap-v3 +pods: + - watcher-merkl-sushiswap-v3