From d1d06e9aed1b3d73c85b10b5123b83dc04bfd632 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 8 Aug 2023 10:40:33 +0530 Subject: [PATCH] Add watcher services --- .../docker-compose-watcher-sushiswap.yml | 153 ++++++++++++++++++ .../watcher-sushiswap/erc20-watcher.toml | 40 +++++ .../watcher-sushiswap/sushi-info-watcher.toml | 91 +++++++++++ .../watcher-sushiswap/sushi-watcher.toml | 42 +++++ .../cerc-watcher-sushiswap/Dockerfile | 11 ++ .../cerc-watcher-sushiswap/build.sh | 9 ++ app/data/container-image-list.txt | 1 + app/data/stacks/sushiswap/stack.yml | 1 + 8 files changed, 348 insertions(+) create mode 100644 app/data/config/watcher-sushiswap/erc20-watcher.toml create mode 100644 app/data/config/watcher-sushiswap/sushi-info-watcher.toml create mode 100644 app/data/config/watcher-sushiswap/sushi-watcher.toml create mode 100644 app/data/container-build/cerc-watcher-sushiswap/Dockerfile create mode 100755 app/data/container-build/cerc-watcher-sushiswap/build.sh diff --git a/app/data/compose/docker-compose-watcher-sushiswap.yml b/app/data/compose/docker-compose-watcher-sushiswap.yml index 946ba6da..285dcaf7 100644 --- a/app/data/compose/docker-compose-watcher-sushiswap.yml +++ b/app/data/compose/docker-compose-watcher-sushiswap.yml @@ -1,6 +1,156 @@ version: '3.2' services: + sushiswap-watcher-db: + restart: unless-stopped + image: postgres:14-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_MULTIPLE_DATABASES=erc20-watcher,sushi-watcher,sushi-info-watcher,erc20-watcher-job-queue,sushi-watcher-job-queue,sushi-info-watcher-job-queue + - POSTGRES_EXTENSION=erc20-watcher-job-queue:pgcrypto,sushi-watcher-job-queue:pgcrypto,sushi-info-watcher-job-queue:pgcrypto + - POSTGRES_PASSWORD=password + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "work_mem=2GB"] + volumes: + - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh + - ../config/postgresql/create-pg-stat-statements.sql:/docker-entrypoint-initdb.d/create-pg-stat-statements.sql + - sushiswap_watcher_db_data:/var/lib/postgresql/data + ports: + - "0.0.0.0:15435:5432" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + shm_size: '8GB' + + erc20-watcher-server: + restart: unless-stopped + depends_on: + sushiswap-watcher-db: + condition: service_healthy + image: cerc/watcher-sushiswap:local + working_dir: /app/packages/erc20-watcher + environment: + - DEBUG=vulcanize:* + command: ["node", "--enable-source-maps", "dist/server.js"] + volumes: + - ../config/watcher-sushiswap/erc20-watcher.toml:/app/packages/erc20-watcher/environments/local.toml + ports: + - "0.0.0.0:3005:3001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3001"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + sushi-watcher-job-runner: + restart: unless-stopped + depends_on: + sushiswap-watcher-db: + condition: service_healthy + image: cerc/watcher-sushiswap:local + working_dir: /app/packages/uni-watcher + environment: + - DEBUG=vulcanize:* + command: ["node", "--enable-source-maps", "dist/job-runner.js"] + volumes: + - ../config/watcher-sushiswap/sushi-watcher.toml:/app/packages/uni-watcher/environments/local.toml + ports: + - "0.0.0.0:9004:9000" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "9000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + sushi-watcher-server: + restart: unless-stopped + depends_on: + sushiswap-watcher-db: + condition: service_healthy + sushi-watcher-job-runner: + condition: service_healthy + image: cerc/watcher-sushiswap:local + environment: + - DEBUG=vulcanize:* + working_dir: /app/packages/uni-watcher + command: ["node", "--enable-source-maps", "dist/server.js"] + volumes: + - ../config/watcher-sushiswap/sushi-watcher.toml:/app/packages/uni-watcher/environments/local.toml + ports: + - "0.0.0.0:3003:3003" + - "0.0.0.0:9005:9001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3003"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + sushi-info-watcher-job-runner: + restart: unless-stopped + depends_on: + sushiswap-watcher-db: + condition: service_healthy + erc20-watcher-server: + condition: service_healthy + sushi-watcher-server: + condition: service_healthy + image: cerc/watcher-sushiswap:local + working_dir: /app/packages/uni-info-watcher + environment: + - DEBUG=vulcanize:* + command: ["node", "--enable-source-maps", "dist/job-runner.js"] + volumes: + - ../config/watcher-sushiswap/sushi-info-watcher.toml:/app/packages/uni-info-watcher/environments/local.toml + ports: + - "0.0.0.0:9006:9002" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "9002"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + sushi-info-watcher-server: + restart: unless-stopped + depends_on: + sushiswap-watcher-db: + condition: service_healthy + erc20-watcher-server: + condition: service_healthy + sushi-watcher-server: + condition: service_healthy + sushi-info-watcher-job-runner: + condition: service_healthy + image: cerc/watcher-sushiswap:local + working_dir: /app/packages/uni-info-watcher + command: ["node", "--enable-source-maps", "dist/server.js"] + volumes: + - ../config/watcher-sushiswap/sushi-info-watcher.toml:/app/packages/uni-info-watcher/environments/local.toml + ports: + - "0.0.0.0:3004:3004" + - "0.0.0.0:9007:9003" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3004"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Deploys the core (UniswapV3Factory) contract sushiswap-v3-core: image: cerc/sushiswap-v3-core:local @@ -12,3 +162,6 @@ services: image: cerc/sushiswap-v3-periphery:local env_file: - ../config/watcher-sushiswap/contract-deployment.env + +volumes: + uniswap_watcher_db_data: diff --git a/app/data/config/watcher-sushiswap/erc20-watcher.toml b/app/data/config/watcher-sushiswap/erc20-watcher.toml new file mode 100644 index 00000000..f8882829 --- /dev/null +++ b/app/data/config/watcher-sushiswap/erc20-watcher.toml @@ -0,0 +1,40 @@ +[server] + host = "0.0.0.0" + port = 3001 + mode = "eth_call" + kind = "lazy" + +[metrics] + host = "127.0.0.1" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "sushiswap-watcher-db" + port = 5432 + database = "erc20-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + maxQueryExecutionTime = 100 + +# TODO: Update to use rpc-eth-client +[upstream] + [upstream.ethServer] + gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql" + rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082" + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@sushiswap-watcher-db:5432/erc20-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 100 + eventsInBatch = 50 + blockDelayInMilliSecs = 2000 diff --git a/app/data/config/watcher-sushiswap/sushi-info-watcher.toml b/app/data/config/watcher-sushiswap/sushi-info-watcher.toml new file mode 100644 index 00000000..3bf3e288 --- /dev/null +++ b/app/data/config/watcher-sushiswap/sushi-info-watcher.toml @@ -0,0 +1,91 @@ +[server] + host = "0.0.0.0" + port = 3004 + mode = "prod" + kind = "active" + + # Checkpointing state. + checkpointing = true + + # Checkpoint interval in number of blocks. + checkpointInterval = 50000 + + # Enable state creation + enableState = false + + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 + + # Interval in number of blocks at which to clear entities cache. + clearEntitiesCacheInterval = 1000 + + # Boolean to skip updating entity fields required in state creation and not required in the frontend. + skipStateFieldsUpdate = false + + # Boolean to load GQL query nested entity relations sequentially. + loadRelationsSequential = false + + # Max GQL API requests to process simultaneously (defaults to 1). + maxSimultaneousRequests = 1 + + # GQL cache settings + [server.gqlCache] + enabled = true + + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize + + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day + +[metrics] + host = "0.0.0.0" + port = 9002 + [metrics.gql] + port = 9003 + +[database] + type = "postgres" + host = "sushiswap-watcher-db" + port = 5432 + database = "sushi-info-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + maxQueryExecutionTime = 100 + + [database.extra] + # maximum number of clients the pool should contain + max = 20 + +# TODO: Update to use rpc-eth-client +[upstream] + [upstream.ethServer] + gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql" + rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082" + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + + [upstream.uniWatcher] + gqlEndpoint = "http://sushi-watcher-server:3003/graphql" + gqlSubscriptionEndpoint = "ws://sushi-watcher-server:3003/graphql" + + [upstream.tokenWatcher] + gqlEndpoint = "http://erc20-watcher-server:3001/graphql" + gqlSubscriptionEndpoint = "ws://erc20-watcher-server:3001/graphql" + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@sushiswap-watcher-db:5432/sushi-info-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 1000 + eventsInBatch = 50 + subgraphEventsOrder = true + blockDelayInMilliSecs = 2000 + prefetchBlocksInMem = true + prefetchBlockCount = 10 diff --git a/app/data/config/watcher-sushiswap/sushi-watcher.toml b/app/data/config/watcher-sushiswap/sushi-watcher.toml new file mode 100644 index 00000000..d386f5b6 --- /dev/null +++ b/app/data/config/watcher-sushiswap/sushi-watcher.toml @@ -0,0 +1,42 @@ +[server] + host = "0.0.0.0" + port = 3003 + kind = "active" + +[metrics] + host = "0.0.0.0" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "sushiswap-watcher-db" + port = 5432 + database = "sushi-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + maxQueryExecutionTime = 100 + +# TODO: Update to use rpc-eth-client +[upstream] + [upstream.ethServer] + gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql" + rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082" + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@sushiswap-watcher-db:5432/sushi-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 0 + eventsInBatch = 50 + lazyUpdateBlockProgress = true + blockDelayInMilliSecs = 2000 + prefetchBlocksInMem = true + prefetchBlockCount = 10 diff --git a/app/data/container-build/cerc-watcher-sushiswap/Dockerfile b/app/data/container-build/cerc-watcher-sushiswap/Dockerfile new file mode 100644 index 00000000..4c2eb15f --- /dev/null +++ b/app/data/container-build/cerc-watcher-sushiswap/Dockerfile @@ -0,0 +1,11 @@ +FROM node:18.15.0-alpine3.16 + +RUN apk --update --no-cache add git python3 alpine-sdk bash + +WORKDIR /app + +COPY . . + +RUN echo "Building uniswap-watcher-ts" && \ + git checkout sushiswap && \ + yarn && yarn build && yarn build:contracts diff --git a/app/data/container-build/cerc-watcher-sushiswap/build.sh b/app/data/container-build/cerc-watcher-sushiswap/build.sh new file mode 100755 index 00000000..1a3762f2 --- /dev/null +++ b/app/data/container-build/cerc-watcher-sushiswap/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Build cerc/watcher-sushiswap + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/watcher-sushiswap:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/uniswap-watcher-ts diff --git a/app/data/container-image-list.txt b/app/data/container-image-list.txt index 4f6534c8..444ee995 100644 --- a/app/data/container-image-list.txt +++ b/app/data/container-image-list.txt @@ -46,3 +46,4 @@ cerc/lasso cerc/reth cerc/sushiswap-v3-core cerc/sushiswap-v3-periphery +cerc/watcher-sushiswap diff --git a/app/data/stacks/sushiswap/stack.yml b/app/data/stacks/sushiswap/stack.yml index b4506cbf..65e685ca 100644 --- a/app/data/stacks/sushiswap/stack.yml +++ b/app/data/stacks/sushiswap/stack.yml @@ -13,6 +13,7 @@ containers: ## sushiswap images - cerc/sushiswap-v3-core - cerc/sushiswap-v3-periphery + - cerc/watcher-sushiswap pods: - fixturenet-lotus - watcher-sushiswap