cerc-io/stack-orchestrator
17
1
Fork 1
Code Issues 110 Pull Requests 12 Actions Packages Projects Releases 473 Wiki Activity

[WIP] Add a Uniswap v3 watcher-only stack (#82)

Browse Source 
* Add a stack for uniswap watcher with info app

* Add deployment instructions

* Use docker interactive option while loading the db dumps
This commit is contained in:
prathamesh0 2022-12-22 19:30:56 +05:30 committed by GitHub
parent 27c3efabe2
commit d51508b01d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 498 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/data/container-image-list.txt
View File

@ -15,6 +15,8 @@ cerc/fixturenet-eth-lighthouse
cerc/watcher-mobymask
cerc/watcher-erc20
cerc/watcher-erc721
cerc/watcher-uniswap-v3
cerc/uniswap-v3-info
cerc/test-container
cerc/eth-probe
cerc/builder-js

1
app/data/pod-list.txt
View File

@ -13,6 +13,7 @@ fixturenet-eth
watcher-mobymask
watcher-erc20
watcher-erc721
watcher-uniswap-v3
test
eth-probe
keycloak

2
app/data/repository-list.txt
View File

@ -11,6 +11,8 @@ cerc-io/laconic-sdk
cerc-io/laconic-registry-cli
cerc-io/mobymask-watcher
cerc-io/watcher-ts
vulcanize/uniswap-watcher-ts
vulcanize/uniswap-v3-info
vulcanize/assemblyscript
cerc-io/eth-probe
cerc-io/tx-spammer

170
compose/docker-compose-watcher-uniswap-v3.yml Normal file
View File

@ -0,0 +1,170 @@
version: '3.2'
services:
uniswap-watcher-db:
restart: unless-stopped
image: postgres:14-alpine
environment:
- POSTGRES_USER=vdbm
- POSTGRES_MULTIPLE_DATABASES=erc20-watcher,uni-watcher,uni-info-watcher,erc20-watcher-job-queue,uni-watcher-job-queue,uni-info-watcher-job-queue
- POSTGRES_EXTENSION=erc20-watcher-job-queue:pgcrypto,uni-watcher-job-queue:pgcrypto,uni-info-watcher-job-queue:pgcrypto
- POSTGRES_PASSWORD=password
command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "work_mem=2GB"]
volumes:
- ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
- ../config/postgresql/create-pg-stat-statements.sql:/docker-entrypoint-initdb.d/create-pg-stat-statements.sql
- uniswap_watcher_db_data:/var/lib/postgresql/data
ports:
- "0.0.0.0:15435:5432"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "5432"]
interval: 20s
timeout: 5s
retries: 15
start_period: 10s
shm_size: '8GB'
erc20-watcher-server:
restart: unless-stopped
depends_on:
uniswap-watcher-db:
condition: service_healthy
image: cerc/watcher-uniswap-v3:local
working_dir: /app/packages/erc20-watcher
environment:
- DEBUG=vulcanize:*
command: ["node", "--enable-source-maps", "dist/server.js"]
volumes:
- ../config/watcher-uniswap-v3/erc20-watcher.toml:/app/packages/erc20-watcher/environments/local.toml
ports:
- "0.0.0.0:3005:3001"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "3001"]
interval: 20s
timeout: 5s
retries: 15
start_period: 5s
extra_hosts:
- "host.docker.internal:host-gateway"
uni-watcher-job-runner:
restart: unless-stopped
depends_on:
uniswap-watcher-db:
condition: service_healthy
image: cerc/watcher-uniswap-v3:local
working_dir: /app/packages/uni-watcher
environment:
- DEBUG=vulcanize:*
command: ["sh", "-c", "./watch-contract.sh && node --enable-source-maps dist/job-runner.js"]
volumes:
- ../config/watcher-uniswap-v3/uni-watcher.toml:/app/packages/uni-watcher/environments/local.toml
- ../config/watcher-uniswap-v3/watch-contract.sh:/app/packages/uni-watcher/watch-contract.sh
ports:
- "0.0.0.0:9004:9000"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "9000"]
interval: 20s
timeout: 5s
retries: 15
start_period: 5s
extra_hosts:
- "host.docker.internal:host-gateway"
uni-watcher-server:
restart: unless-stopped
depends_on:
uniswap-watcher-db:
condition: service_healthy
uni-watcher-job-runner:
condition: service_healthy
image: cerc/watcher-uniswap-v3:local
environment:
- UNISWAP_START_BLOCK=12369621
- DEBUG=vulcanize:*
working_dir: /app/packages/uni-watcher
command: ["./run.sh"]
volumes:
- ../config/watcher-uniswap-v3/uni-watcher.toml:/app/packages/uni-watcher/environments/local.toml
- ../config/watcher-uniswap-v3/run.sh:/app/packages/uni-watcher/run.sh
ports:
- "0.0.0.0:3003:3003"
- "0.0.0.0:9005:9001"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "3003"]
interval: 20s
timeout: 5s
retries: 15
start_period: 5s
extra_hosts:
- "host.docker.internal:host-gateway"
uni-info-watcher-job-runner:
restart: unless-stopped
depends_on:
uniswap-watcher-db:
condition: service_healthy
erc20-watcher-server:
condition: service_healthy
uni-watcher-server:
condition: service_healthy
image: cerc/watcher-uniswap-v3:local
working_dir: /app/packages/uni-info-watcher
environment:
- DEBUG=vulcanize:*
command: ["node", "--enable-source-maps", "dist/job-runner.js"]
volumes:
- ../config/watcher-uniswap-v3/uni-info-watcher.toml:/app/packages/uni-info-watcher/environments/local.toml
ports:
- "0.0.0.0:9006:9002"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "9002"]
interval: 20s
timeout: 5s
retries: 15
start_period: 5s
extra_hosts:
- "host.docker.internal:host-gateway"
uni-info-watcher-server:
restart: unless-stopped
depends_on:
uniswap-watcher-db:
condition: service_healthy
erc20-watcher-server:
condition: service_healthy
uni-watcher-server:
condition: service_healthy
uni-info-watcher-job-runner:
condition: service_healthy
image: cerc/watcher-uniswap-v3:local
environment:
- UNISWAP_START_BLOCK=12369621
working_dir: /app/packages/uni-info-watcher
command: ["./run.sh"]
volumes:
- ../config/watcher-uniswap-v3/uni-info-watcher.toml:/app/packages/uni-info-watcher/environments/local.toml
- ../config/watcher-uniswap-v3/run.sh:/app/packages/uni-info-watcher/run.sh
ports:
- "0.0.0.0:3004:3004"
- "0.0.0.0:9007:9003"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "3004"]
interval: 20s
timeout: 5s
retries: 15
start_period: 5s
extra_hosts:
- "host.docker.internal:host-gateway"
uniswap-v3-info:
depends_on:
uni-info-watcher-server:
condition: service_healthy
image: cerc/uniswap-v3-info:local
ports:
- "0.0.0.0:3006:3000"
volumes:
uniswap_watcher_db_data:

1
config/postgresql/create-pg-stat-statements.sql Normal file
View File

@ -0,0 +1 @@
CREATE EXTENSION pg_stat_statements;

39
config/watcher-uniswap-v3/erc20-watcher.toml Normal file
View File

@ -0,0 +1,39 @@
[server]
host = "0.0.0.0"
port = 3001
mode = "eth_call"
kind = "lazy"
[metrics]
host = "127.0.0.1"
port = 9000
[metrics.gql]
port = 9001
[database]
type = "postgres"
host = "uniswap-watcher-db"
port = 5432
database = "erc20-watcher"
username = "vdbm"
password = "password"
synchronize = true
logging = false
maxQueryExecutionTime = 100
[upstream]
[upstream.ethServer]
gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql"
rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082"
[upstream.cache]
name = "requests"
enabled = false
deleteOnStart = false
[jobQueue]
dbConnectionString = "postgres://vdbm:password@uniswap-watcher-db:5432/erc20-watcher-job-queue"
maxCompletionLagInSecs = 300
jobDelayInMilliSecs = 100
eventsInBatch = 50
blockDelayInMilliSecs = 2000

10
config/watcher-uniswap-v3/run.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/sh
set -e
set -u
echo "Initializing watcher..."
yarn fill --start-block $UNISWAP_START_BLOCK --end-block $((UNISWAP_START_BLOCK + 1))
echo "Running active server"
DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js

90
config/watcher-uniswap-v3/uni-info-watcher.toml Normal file
View File

@ -0,0 +1,90 @@
[server]
host = "0.0.0.0"
port = 3004
mode = "prod"
kind = "active"
# Checkpointing state.
checkpointing = true
# Checkpoint interval in number of blocks.
checkpointInterval = 50000
# Enable state creation
enableState = false
# Max block range for which to return events in eventsInRange GQL query.
# Use -1 for skipping check on block range.
maxEventsBlockRange = 1000
# Interval in number of blocks at which to clear entities cache.
clearEntitiesCacheInterval = 1000
# Boolean to skip updating entity fields required in state creation and not required in the frontend.
skipStateFieldsUpdate = false
# Boolean to load GQL query nested entity relations sequentially.
loadRelationsSequential = false
# Max GQL API requests to process simultaneously (defaults to 1).
maxSimultaneousRequests = 1
# GQL cache settings
[server.gqlCache]
enabled = true
# Max in-memory cache size (in bytes) (default 8 MB)
# maxCacheSize
# GQL cache-control max-age settings (in seconds)
maxAge = 15
timeTravelMaxAge = 86400 # 1 day
[metrics]
host = "0.0.0.0"
port = 9002
[metrics.gql]
port = 9003
[database]
type = "postgres"
host = "uniswap-watcher-db"
port = 5432
database = "uni-info-watcher"
username = "vdbm"
password = "password"
synchronize = true
logging = false
maxQueryExecutionTime = 100
[database.extra]
# maximum number of clients the pool should contain
max = 20
[upstream]
[upstream.ethServer]
gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql"
rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082"
[upstream.cache]
name = "requests"
enabled = false
deleteOnStart = false
[upstream.uniWatcher]
gqlEndpoint = "http://uni-watcher-server:3003/graphql"
gqlSubscriptionEndpoint = "ws://uni-watcher-server:3003/graphql"
[upstream.tokenWatcher]
gqlEndpoint = "http://erc20-watcher-server:3001/graphql"
gqlSubscriptionEndpoint = "ws://erc20-watcher-server:3001/graphql"
[jobQueue]
dbConnectionString = "postgres://vdbm:password@uniswap-watcher-db:5432/uni-info-watcher-job-queue"
maxCompletionLagInSecs = 300
jobDelayInMilliSecs = 1000
eventsInBatch = 50
subgraphEventsOrder = true
blockDelayInMilliSecs = 2000
prefetchBlocksInMem = true
prefetchBlockCount = 10

41
config/watcher-uniswap-v3/uni-watcher.toml Normal file
View File

@ -0,0 +1,41 @@
[server]
host = "0.0.0.0"
port = 3003
kind = "active"
[metrics]
host = "0.0.0.0"
port = 9000
[metrics.gql]
port = 9001
[database]
type = "postgres"
host = "uniswap-watcher-db"
port = 5432
database = "uni-watcher"
username = "vdbm"
password = "password"
synchronize = true
logging = false
maxQueryExecutionTime = 100
[upstream]
[upstream.ethServer]
gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql"
rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082"
[upstream.cache]
name = "requests"
enabled = false
deleteOnStart = false
[jobQueue]
dbConnectionString = "postgres://vdbm:password@uniswap-watcher-db:5432/uni-watcher-job-queue"
maxCompletionLagInSecs = 300
jobDelayInMilliSecs = 0
eventsInBatch = 50
lazyUpdateBlockProgress = true
blockDelayInMilliSecs = 2000
prefetchBlocksInMem = true
prefetchBlockCount = 10

10
config/watcher-uniswap-v3/watch-contract.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/sh
set -e
set -u
echo "Watching factory contract 0x1F98431c8aD98523631AE4a59f267346ea31F984"
yarn watch:contract --address 0x1F98431c8aD98523631AE4a59f267346ea31F984 --kind factory --startingBlock 12369621 --checkpoint
echo "Watching nfpm contract 0xC36442b4a4522E871399CD717aBDD847Ab11FE88"
yarn watch:contract --address 0xC36442b4a4522E871399CD717aBDD847Ab11FE88 --kind nfpm --startingBlock 12369651 --checkpoint

13
container-build/cerc-uniswap-v3-info/Dockerfile Normal file
View File

@ -0,0 +1,13 @@
FROM node:15.3.0-alpine3.10
RUN apk --update --no-cache add make git
WORKDIR /app
COPY . .
RUN echo "Building uniswap-v3-info" && \
git checkout v0.1.1 && \
yarn
CMD ["sh", "-c", "yarn start"]

7
container-build/cerc-uniswap-v3-info/build.sh Executable file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env bash
# Build cerc/uniswap-v3-info
# See: https://stackoverflow.com/a/246128/1701505
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
docker build -t cerc/uniswap-v3-info:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/uniswap-v3-info

11
container-build/cerc-watcher-uniswap-v3/Dockerfile Normal file
View File

@ -0,0 +1,11 @@
FROM node:16.17.1-alpine3.16
RUN apk --update --no-cache add git python3 alpine-sdk
WORKDIR /app
COPY . .
RUN echo "Building uniswap-watcher-ts" && \
git checkout v0.3.4 && \
yarn && yarn build && yarn build:contracts

7
container-build/cerc-watcher-uniswap-v3/build.sh Executable file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env bash
# Build cerc/watcher-uniswap-v3
# See: https://stackoverflow.com/a/246128/1701505
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
docker build -t cerc/watcher-uniswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/uniswap-watcher-ts

2
stacks/erc20/README.md
View File

@ -44,7 +44,7 @@ Instructions to deploy a local ERC20 watcher stack (core + watcher) for demonstr
$ export TOKEN_ADDRESS=<TOKEN_ADDRESS>
```
* Open `http://localhost:3001/graphql` (GraphQL Playground) in a browser window
* Open `http://localhost:3002/graphql` (GraphQL Playground) in a browser window
* Connect MetaMask to `http://localhost:8545` (with chain ID `99`)

83
stacks/uniswap-v3/README.md Normal file
View File

@ -0,0 +1,83 @@
# Uniswap v3
Instructions to deploy Uniswap v3 watcher stack (watcher + uniswap-v3-info frontend app) using [laconic-stack-orchestrator](../../README.md#setup)
## Prerequisites
* Access to [uniswap-watcher-ts](https://github.com/vulcanize/uniswap-watcher-ts).
* This deployment expects core services to be running; specifically, it requires `ipld-eth-server` RPC and GQL endpoints. Update the `upstream.ethServer` endpoints in the [watcher config files](../../config/watcher-uniswap-v3) accordingly:
```toml
[upstream]
[upstream.ethServer]
gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql"
rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082"
```
* `uni-watcher` and `uni-info-watcher` database dumps (optional).
## Setup
* Clone / pull required repositories:
```bash
$ laconic-so setup-repositories --include vulcanize/uniswap-watcher-ts,vulcanize/uniswap-v3-info --git-ssh --pull
```
* Build watcher and info app container images:
```bash
$ laconic-so build-containers --include cerc/watcher-uniswap-v3,cerc/uniswap-v3-info
```
This should create the required docker images in the local image registry.
## Deploy
* (Optional) Initialize the watcher database with existing database dumps if available:
* Start the watcher database to be initialized:
```bash
$ laconic-so deploy-system --include watcher-uniswap-v3 up uniswap-watcher-db
```
* Find the watcher database container's id using `docker ps` and export it for further usage:
```bash
$ export CONTAINER_ID=<CONTAINER_ID>
```
* Load watcher database dumps:
```bash
# uni-watcher database
$ docker exec -i $CONTAINER_ID psql -U vdbm uni-watcher < UNI_WATCHER_DB_DUMP_FILE_PATH.sql
# uni-info-watcher database
$ docker exec -i $CONTAINER_ID psql -U vdbm uni-info-watcher < UNI_INFO_WATCHER_DB_DUMP_FILE_PATH.sql
```
* Start all the watcher and info app services:
```bash
$ laconic-so deploy-system --include watcher-uniswap-v3 up
```
* Check that all the services are up and healthy:
```bash
$ docker ps
```
* The `uni-info-watcher` GraphQL Playground can be accessed at `http://localhost:3004/graphql`
* The frontend app can be accessed at `http://localhost:3006`
## Clean up
* To stop all the services running in background:
```bash
$ laconic-so deploy-system --include watcher-uniswap-v3 down
```

10
stacks/uniswap-v3/stack.yml Normal file
View File

@ -0,0 +1,10 @@
version: "1.0"
name: uniswap-v3
repos:
- vulcanize/uniswap-watcher-ts
- vulcanize/uniswap-v3-info
containers:
- cerc/watcher-uniswap-v3
- cerc/uniswap-v3-info
pods:
- watcher-uniswap-v3