diff --git a/app/data/container-image-list.txt b/app/data/container-image-list.txt index 4c7c052..c697fe2 100644 --- a/app/data/container-image-list.txt +++ b/app/data/container-image-list.txt @@ -15,6 +15,8 @@ cerc/fixturenet-eth-lighthouse cerc/watcher-mobymask cerc/watcher-erc20 cerc/watcher-erc721 +cerc/watcher-uniswap-v3 +cerc/uniswap-v3-info cerc/test-container cerc/eth-probe cerc/builder-js diff --git a/app/data/pod-list.txt b/app/data/pod-list.txt index 42c46ac..e5cadd5 100644 --- a/app/data/pod-list.txt +++ b/app/data/pod-list.txt @@ -13,6 +13,7 @@ fixturenet-eth watcher-mobymask watcher-erc20 watcher-erc721 +watcher-uniswap-v3 test eth-probe keycloak diff --git a/app/data/repository-list.txt b/app/data/repository-list.txt index c73ce29..cdf40b2 100644 --- a/app/data/repository-list.txt +++ b/app/data/repository-list.txt @@ -11,6 +11,8 @@ cerc-io/laconic-sdk cerc-io/laconic-registry-cli cerc-io/mobymask-watcher cerc-io/watcher-ts +vulcanize/uniswap-watcher-ts +vulcanize/uniswap-v3-info vulcanize/assemblyscript cerc-io/eth-probe cerc-io/tx-spammer diff --git a/compose/docker-compose-watcher-uniswap-v3.yml b/compose/docker-compose-watcher-uniswap-v3.yml new file mode 100644 index 0000000..b98f6a7 --- /dev/null +++ b/compose/docker-compose-watcher-uniswap-v3.yml @@ -0,0 +1,170 @@ +version: '3.2' + +services: + + uniswap-watcher-db: + restart: unless-stopped + image: postgres:14-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_MULTIPLE_DATABASES=erc20-watcher,uni-watcher,uni-info-watcher,erc20-watcher-job-queue,uni-watcher-job-queue,uni-info-watcher-job-queue + - POSTGRES_EXTENSION=erc20-watcher-job-queue:pgcrypto,uni-watcher-job-queue:pgcrypto,uni-info-watcher-job-queue:pgcrypto + - POSTGRES_PASSWORD=password + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "work_mem=2GB"] + volumes: + - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh + - ../config/postgresql/create-pg-stat-statements.sql:/docker-entrypoint-initdb.d/create-pg-stat-statements.sql + - uniswap_watcher_db_data:/var/lib/postgresql/data + ports: + - "0.0.0.0:15435:5432" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + shm_size: '8GB' + + erc20-watcher-server: + restart: unless-stopped + depends_on: + uniswap-watcher-db: + condition: service_healthy + image: cerc/watcher-uniswap-v3:local + working_dir: /app/packages/erc20-watcher + environment: + - DEBUG=vulcanize:* + command: ["node", "--enable-source-maps", "dist/server.js"] + volumes: + - ../config/watcher-uniswap-v3/erc20-watcher.toml:/app/packages/erc20-watcher/environments/local.toml + ports: + - "0.0.0.0:3005:3001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3001"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + uni-watcher-job-runner: + restart: unless-stopped + depends_on: + uniswap-watcher-db: + condition: service_healthy + image: cerc/watcher-uniswap-v3:local + working_dir: /app/packages/uni-watcher + environment: + - DEBUG=vulcanize:* + command: ["sh", "-c", "./watch-contract.sh && node --enable-source-maps dist/job-runner.js"] + volumes: + - ../config/watcher-uniswap-v3/uni-watcher.toml:/app/packages/uni-watcher/environments/local.toml + - ../config/watcher-uniswap-v3/watch-contract.sh:/app/packages/uni-watcher/watch-contract.sh + ports: + - "0.0.0.0:9004:9000" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "9000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + uni-watcher-server: + restart: unless-stopped + depends_on: + uniswap-watcher-db: + condition: service_healthy + uni-watcher-job-runner: + condition: service_healthy + image: cerc/watcher-uniswap-v3:local + environment: + - UNISWAP_START_BLOCK=12369621 + - DEBUG=vulcanize:* + working_dir: /app/packages/uni-watcher + command: ["./run.sh"] + volumes: + - ../config/watcher-uniswap-v3/uni-watcher.toml:/app/packages/uni-watcher/environments/local.toml + - ../config/watcher-uniswap-v3/run.sh:/app/packages/uni-watcher/run.sh + ports: + - "0.0.0.0:3003:3003" + - "0.0.0.0:9005:9001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3003"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + uni-info-watcher-job-runner: + restart: unless-stopped + depends_on: + uniswap-watcher-db: + condition: service_healthy + erc20-watcher-server: + condition: service_healthy + uni-watcher-server: + condition: service_healthy + image: cerc/watcher-uniswap-v3:local + working_dir: /app/packages/uni-info-watcher + environment: + - DEBUG=vulcanize:* + command: ["node", "--enable-source-maps", "dist/job-runner.js"] + volumes: + - ../config/watcher-uniswap-v3/uni-info-watcher.toml:/app/packages/uni-info-watcher/environments/local.toml + ports: + - "0.0.0.0:9006:9002" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "9002"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + uni-info-watcher-server: + restart: unless-stopped + depends_on: + uniswap-watcher-db: + condition: service_healthy + erc20-watcher-server: + condition: service_healthy + uni-watcher-server: + condition: service_healthy + uni-info-watcher-job-runner: + condition: service_healthy + image: cerc/watcher-uniswap-v3:local + environment: + - UNISWAP_START_BLOCK=12369621 + working_dir: /app/packages/uni-info-watcher + command: ["./run.sh"] + volumes: + - ../config/watcher-uniswap-v3/uni-info-watcher.toml:/app/packages/uni-info-watcher/environments/local.toml + - ../config/watcher-uniswap-v3/run.sh:/app/packages/uni-info-watcher/run.sh + ports: + - "0.0.0.0:3004:3004" + - "0.0.0.0:9007:9003" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3004"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + uniswap-v3-info: + depends_on: + uni-info-watcher-server: + condition: service_healthy + image: cerc/uniswap-v3-info:local + ports: + - "0.0.0.0:3006:3000" + +volumes: + uniswap_watcher_db_data: diff --git a/config/postgresql/create-pg-stat-statements.sql b/config/postgresql/create-pg-stat-statements.sql new file mode 100644 index 0000000..d3c6679 --- /dev/null +++ b/config/postgresql/create-pg-stat-statements.sql @@ -0,0 +1 @@ +CREATE EXTENSION pg_stat_statements; diff --git a/config/watcher-uniswap-v3/erc20-watcher.toml b/config/watcher-uniswap-v3/erc20-watcher.toml new file mode 100644 index 0000000..3e258ee --- /dev/null +++ b/config/watcher-uniswap-v3/erc20-watcher.toml @@ -0,0 +1,39 @@ +[server] + host = "0.0.0.0" + port = 3001 + mode = "eth_call" + kind = "lazy" + +[metrics] + host = "127.0.0.1" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "uniswap-watcher-db" + port = 5432 + database = "erc20-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + maxQueryExecutionTime = 100 + +[upstream] + [upstream.ethServer] + gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql" + rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082" + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@uniswap-watcher-db:5432/erc20-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 100 + eventsInBatch = 50 + blockDelayInMilliSecs = 2000 diff --git a/config/watcher-uniswap-v3/run.sh b/config/watcher-uniswap-v3/run.sh new file mode 100755 index 0000000..d8dcffb --- /dev/null +++ b/config/watcher-uniswap-v3/run.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -e +set -u + +echo "Initializing watcher..." +yarn fill --start-block $UNISWAP_START_BLOCK --end-block $((UNISWAP_START_BLOCK + 1)) + +echo "Running active server" +DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js diff --git a/config/watcher-uniswap-v3/uni-info-watcher.toml b/config/watcher-uniswap-v3/uni-info-watcher.toml new file mode 100644 index 0000000..2f09284 --- /dev/null +++ b/config/watcher-uniswap-v3/uni-info-watcher.toml @@ -0,0 +1,90 @@ +[server] + host = "0.0.0.0" + port = 3004 + mode = "prod" + kind = "active" + + # Checkpointing state. + checkpointing = true + + # Checkpoint interval in number of blocks. + checkpointInterval = 50000 + + # Enable state creation + enableState = false + + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 + + # Interval in number of blocks at which to clear entities cache. + clearEntitiesCacheInterval = 1000 + + # Boolean to skip updating entity fields required in state creation and not required in the frontend. + skipStateFieldsUpdate = false + + # Boolean to load GQL query nested entity relations sequentially. + loadRelationsSequential = false + + # Max GQL API requests to process simultaneously (defaults to 1). + maxSimultaneousRequests = 1 + + # GQL cache settings + [server.gqlCache] + enabled = true + + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize + + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day + +[metrics] + host = "0.0.0.0" + port = 9002 + [metrics.gql] + port = 9003 + +[database] + type = "postgres" + host = "uniswap-watcher-db" + port = 5432 + database = "uni-info-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + maxQueryExecutionTime = 100 + + [database.extra] + # maximum number of clients the pool should contain + max = 20 + +[upstream] + [upstream.ethServer] + gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql" + rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082" + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + + [upstream.uniWatcher] + gqlEndpoint = "http://uni-watcher-server:3003/graphql" + gqlSubscriptionEndpoint = "ws://uni-watcher-server:3003/graphql" + + [upstream.tokenWatcher] + gqlEndpoint = "http://erc20-watcher-server:3001/graphql" + gqlSubscriptionEndpoint = "ws://erc20-watcher-server:3001/graphql" + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@uniswap-watcher-db:5432/uni-info-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 1000 + eventsInBatch = 50 + subgraphEventsOrder = true + blockDelayInMilliSecs = 2000 + prefetchBlocksInMem = true + prefetchBlockCount = 10 diff --git a/config/watcher-uniswap-v3/uni-watcher.toml b/config/watcher-uniswap-v3/uni-watcher.toml new file mode 100644 index 0000000..bc583e9 --- /dev/null +++ b/config/watcher-uniswap-v3/uni-watcher.toml @@ -0,0 +1,41 @@ +[server] + host = "0.0.0.0" + port = 3003 + kind = "active" + +[metrics] + host = "0.0.0.0" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "uniswap-watcher-db" + port = 5432 + database = "uni-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + maxQueryExecutionTime = 100 + +[upstream] + [upstream.ethServer] + gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql" + rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082" + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@uniswap-watcher-db:5432/uni-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 0 + eventsInBatch = 50 + lazyUpdateBlockProgress = true + blockDelayInMilliSecs = 2000 + prefetchBlocksInMem = true + prefetchBlockCount = 10 diff --git a/config/watcher-uniswap-v3/watch-contract.sh b/config/watcher-uniswap-v3/watch-contract.sh new file mode 100755 index 0000000..aaed7e1 --- /dev/null +++ b/config/watcher-uniswap-v3/watch-contract.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -e +set -u + +echo "Watching factory contract 0x1F98431c8aD98523631AE4a59f267346ea31F984" +yarn watch:contract --address 0x1F98431c8aD98523631AE4a59f267346ea31F984 --kind factory --startingBlock 12369621 --checkpoint + +echo "Watching nfpm contract 0xC36442b4a4522E871399CD717aBDD847Ab11FE88" +yarn watch:contract --address 0xC36442b4a4522E871399CD717aBDD847Ab11FE88 --kind nfpm --startingBlock 12369651 --checkpoint diff --git a/container-build/cerc-uniswap-v3-info/Dockerfile b/container-build/cerc-uniswap-v3-info/Dockerfile new file mode 100644 index 0000000..a01e2ed --- /dev/null +++ b/container-build/cerc-uniswap-v3-info/Dockerfile @@ -0,0 +1,13 @@ +FROM node:15.3.0-alpine3.10 + +RUN apk --update --no-cache add make git + +WORKDIR /app + +COPY . . + +RUN echo "Building uniswap-v3-info" && \ + git checkout v0.1.1 && \ + yarn + +CMD ["sh", "-c", "yarn start"] diff --git a/container-build/cerc-uniswap-v3-info/build.sh b/container-build/cerc-uniswap-v3-info/build.sh new file mode 100755 index 0000000..bda1bb5 --- /dev/null +++ b/container-build/cerc-uniswap-v3-info/build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Build cerc/uniswap-v3-info + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/uniswap-v3-info:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/uniswap-v3-info diff --git a/container-build/cerc-watcher-uniswap-v3/Dockerfile b/container-build/cerc-watcher-uniswap-v3/Dockerfile new file mode 100644 index 0000000..f400755 --- /dev/null +++ b/container-build/cerc-watcher-uniswap-v3/Dockerfile @@ -0,0 +1,11 @@ +FROM node:16.17.1-alpine3.16 + +RUN apk --update --no-cache add git python3 alpine-sdk + +WORKDIR /app + +COPY . . + +RUN echo "Building uniswap-watcher-ts" && \ + git checkout v0.3.4 && \ + yarn && yarn build && yarn build:contracts diff --git a/container-build/cerc-watcher-uniswap-v3/build.sh b/container-build/cerc-watcher-uniswap-v3/build.sh new file mode 100755 index 0000000..432feb0 --- /dev/null +++ b/container-build/cerc-watcher-uniswap-v3/build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Build cerc/watcher-uniswap-v3 + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/watcher-uniswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/uniswap-watcher-ts diff --git a/stacks/erc20/README.md b/stacks/erc20/README.md index eec3d9f..3405b70 100644 --- a/stacks/erc20/README.md +++ b/stacks/erc20/README.md @@ -44,7 +44,7 @@ Instructions to deploy a local ERC20 watcher stack (core + watcher) for demonstr $ export TOKEN_ADDRESS= ``` -* Open `http://localhost:3001/graphql` (GraphQL Playground) in a browser window +* Open `http://localhost:3002/graphql` (GraphQL Playground) in a browser window * Connect MetaMask to `http://localhost:8545` (with chain ID `99`) diff --git a/stacks/uniswap-v3/README.md b/stacks/uniswap-v3/README.md new file mode 100644 index 0000000..99f5505 --- /dev/null +++ b/stacks/uniswap-v3/README.md @@ -0,0 +1,83 @@ +# Uniswap v3 + +Instructions to deploy Uniswap v3 watcher stack (watcher + uniswap-v3-info frontend app) using [laconic-stack-orchestrator](../../README.md#setup) + +## Prerequisites + +* Access to [uniswap-watcher-ts](https://github.com/vulcanize/uniswap-watcher-ts). + +* This deployment expects core services to be running; specifically, it requires `ipld-eth-server` RPC and GQL endpoints. Update the `upstream.ethServer` endpoints in the [watcher config files](../../config/watcher-uniswap-v3) accordingly: + + ```toml + [upstream] + [upstream.ethServer] + gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql" + rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082" + ``` + +* `uni-watcher` and `uni-info-watcher` database dumps (optional). + +## Setup + +* Clone / pull required repositories: + + ```bash + $ laconic-so setup-repositories --include vulcanize/uniswap-watcher-ts,vulcanize/uniswap-v3-info --git-ssh --pull + ``` + +* Build watcher and info app container images: + + ```bash + $ laconic-so build-containers --include cerc/watcher-uniswap-v3,cerc/uniswap-v3-info + ``` + + This should create the required docker images in the local image registry. + +## Deploy + +* (Optional) Initialize the watcher database with existing database dumps if available: + + * Start the watcher database to be initialized: + + ```bash + $ laconic-so deploy-system --include watcher-uniswap-v3 up uniswap-watcher-db + ``` + + * Find the watcher database container's id using `docker ps` and export it for further usage: + + ```bash + $ export CONTAINER_ID= + ``` + + * Load watcher database dumps: + + ```bash + # uni-watcher database + $ docker exec -i $CONTAINER_ID psql -U vdbm uni-watcher < UNI_WATCHER_DB_DUMP_FILE_PATH.sql + + # uni-info-watcher database + $ docker exec -i $CONTAINER_ID psql -U vdbm uni-info-watcher < UNI_INFO_WATCHER_DB_DUMP_FILE_PATH.sql + ``` + +* Start all the watcher and info app services: + + ```bash + $ laconic-so deploy-system --include watcher-uniswap-v3 up + ``` + +* Check that all the services are up and healthy: + + ```bash + $ docker ps + ``` + + * The `uni-info-watcher` GraphQL Playground can be accessed at `http://localhost:3004/graphql` + * The frontend app can be accessed at `http://localhost:3006` + +## Clean up + +* To stop all the services running in background: + + ```bash + $ laconic-so deploy-system --include watcher-uniswap-v3 down + ``` diff --git a/stacks/uniswap-v3/stack.yml b/stacks/uniswap-v3/stack.yml new file mode 100644 index 0000000..8eec8f8 --- /dev/null +++ b/stacks/uniswap-v3/stack.yml @@ -0,0 +1,10 @@ +version: "1.0" +name: uniswap-v3 +repos: + - vulcanize/uniswap-watcher-ts + - vulcanize/uniswap-v3-info +containers: + - cerc/watcher-uniswap-v3 + - cerc/uniswap-v3-info +pods: + - watcher-uniswap-v3