[WIP] Add a Uniswap v3 watcher-only stack (#82)

* Add a stack for uniswap watcher with info app

* Add deployment instructions

* Use docker interactive option while loading the db dumps

app/data/container-image-list.txt

@@ -15,6 +15,8 @@ cerc/fixturenet-eth-lighthouse
 cerc/watcher-mobymask
 cerc/watcher-erc20
 cerc/watcher-erc721
+cerc/watcher-uniswap-v3
+cerc/uniswap-v3-info
 cerc/test-container
 cerc/eth-probe
 cerc/builder-js

app/data/pod-list.txt

@@ -13,6 +13,7 @@ fixturenet-eth
 watcher-mobymask
 watcher-erc20
 watcher-erc721
+watcher-uniswap-v3
 test
 eth-probe
 keycloak

app/data/repository-list.txt

@@ -11,6 +11,8 @@ cerc-io/laconic-sdk
 cerc-io/laconic-registry-cli
 cerc-io/mobymask-watcher
 cerc-io/watcher-ts
+vulcanize/uniswap-watcher-ts
+vulcanize/uniswap-v3-info
 vulcanize/assemblyscript
 cerc-io/eth-probe
 cerc-io/tx-spammer

compose/docker-compose-watcher-uniswap-v3.yml

@@ -0,0 +1,170 @@
+version: '3.2'
+
+services:
+
+  uniswap-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=erc20-watcher,uni-watcher,uni-info-watcher,erc20-watcher-job-queue,uni-watcher-job-queue,uni-info-watcher-job-queue
+      - POSTGRES_EXTENSION=erc20-watcher-job-queue:pgcrypto,uni-watcher-job-queue:pgcrypto,uni-info-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "work_mem=2GB"]
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - ../config/postgresql/create-pg-stat-statements.sql:/docker-entrypoint-initdb.d/create-pg-stat-statements.sql
+      - uniswap_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "0.0.0.0:15435:5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+    shm_size: '8GB'
+
+  erc20-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      uniswap-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-uniswap-v3:local
+    working_dir: /app/packages/erc20-watcher
+    environment:
+      - DEBUG=vulcanize:*
+    command: ["node", "--enable-source-maps", "dist/server.js"]
+    volumes:
+      - ../config/watcher-uniswap-v3/erc20-watcher.toml:/app/packages/erc20-watcher/environments/local.toml
+    ports:
+      - "0.0.0.0:3005:3001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3001"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  uni-watcher-job-runner:
+    restart: unless-stopped
+    depends_on:
+      uniswap-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-uniswap-v3:local
+    working_dir: /app/packages/uni-watcher
+    environment:
+      - DEBUG=vulcanize:*
+    command: ["sh", "-c", "./watch-contract.sh && node --enable-source-maps dist/job-runner.js"]
+    volumes:
+      - ../config/watcher-uniswap-v3/uni-watcher.toml:/app/packages/uni-watcher/environments/local.toml
+      - ../config/watcher-uniswap-v3/watch-contract.sh:/app/packages/uni-watcher/watch-contract.sh
+    ports:
+      - "0.0.0.0:9004:9000"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  uni-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      uniswap-watcher-db:
+        condition: service_healthy
+      uni-watcher-job-runner:
+        condition: service_healthy
+    image: cerc/watcher-uniswap-v3:local
+    environment:
+      - UNISWAP_START_BLOCK=12369621
+      - DEBUG=vulcanize:*
+    working_dir: /app/packages/uni-watcher
+    command: ["./run.sh"]
+    volumes:
+      - ../config/watcher-uniswap-v3/uni-watcher.toml:/app/packages/uni-watcher/environments/local.toml
+      - ../config/watcher-uniswap-v3/run.sh:/app/packages/uni-watcher/run.sh
+    ports:
+      - "0.0.0.0:3003:3003"
+      - "0.0.0.0:9005:9001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3003"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  uni-info-watcher-job-runner:
+    restart: unless-stopped
+    depends_on:
+      uniswap-watcher-db:
+        condition: service_healthy
+      erc20-watcher-server:
+        condition: service_healthy
+      uni-watcher-server:
+        condition: service_healthy
+    image: cerc/watcher-uniswap-v3:local
+    working_dir: /app/packages/uni-info-watcher
+    environment:
+      - DEBUG=vulcanize:*
+    command: ["node", "--enable-source-maps", "dist/job-runner.js"]
+    volumes:
+      - ../config/watcher-uniswap-v3/uni-info-watcher.toml:/app/packages/uni-info-watcher/environments/local.toml
+    ports:
+      - "0.0.0.0:9006:9002"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "9002"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  uni-info-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      uniswap-watcher-db:
+        condition: service_healthy
+      erc20-watcher-server:
+        condition: service_healthy
+      uni-watcher-server:
+        condition: service_healthy
+      uni-info-watcher-job-runner:
+        condition: service_healthy
+    image: cerc/watcher-uniswap-v3:local
+    environment:
+      - UNISWAP_START_BLOCK=12369621
+    working_dir: /app/packages/uni-info-watcher
+    command: ["./run.sh"]
+    volumes:
+      - ../config/watcher-uniswap-v3/uni-info-watcher.toml:/app/packages/uni-info-watcher/environments/local.toml
+      - ../config/watcher-uniswap-v3/run.sh:/app/packages/uni-info-watcher/run.sh
+    ports:
+      - "0.0.0.0:3004:3004"
+      - "0.0.0.0:9007:9003"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3004"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  uniswap-v3-info:
+    depends_on:
+      uni-info-watcher-server:
+        condition: service_healthy
+    image: cerc/uniswap-v3-info:local
+    ports:
+      - "0.0.0.0:3006:3000"
+
+volumes:
+  uniswap_watcher_db_data:

config/postgresql/create-pg-stat-statements.sql

@@ -0,0 +1 @@
+CREATE EXTENSION pg_stat_statements;

config/watcher-uniswap-v3/erc20-watcher.toml

@@ -0,0 +1,39 @@
+[server]
+  host = "0.0.0.0"
+  port = 3001
+  mode = "eth_call"
+  kind = "lazy"
+
+[metrics]
+  host = "127.0.0.1"
+  port = 9000
+    [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "uniswap-watcher-db"
+  port = 5432
+  database = "erc20-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+  maxQueryExecutionTime = 100
+
+[upstream]
+  [upstream.ethServer]
+    gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql"
+    rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082"
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@uniswap-watcher-db:5432/erc20-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50
+  blockDelayInMilliSecs = 2000

config/watcher-uniswap-v3/run.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+set -u
+
+echo "Initializing watcher..."
+yarn fill --start-block $UNISWAP_START_BLOCK --end-block $((UNISWAP_START_BLOCK + 1))
+
+echo "Running active server"
+DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js

config/watcher-uniswap-v3/uni-info-watcher.toml

@@ -0,0 +1,90 @@
+[server]
+  host = "0.0.0.0"
+  port = 3004
+  mode = "prod"
+  kind = "active"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 50000
+
+  # Enable state creation
+  enableState = false
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = 1000
+
+  # Interval in number of blocks at which to clear entities cache.
+  clearEntitiesCacheInterval = 1000
+
+  # Boolean to skip updating entity fields required in state creation and not required in the frontend.
+  skipStateFieldsUpdate = false
+
+  # Boolean to load GQL query nested entity relations sequentially.
+  loadRelationsSequential = false
+
+  # Max GQL API requests to process simultaneously (defaults to 1).
+  maxSimultaneousRequests = 1
+
+  # GQL cache settings
+  [server.gqlCache]
+    enabled = true
+
+    # Max in-memory cache size (in bytes) (default 8 MB)
+    # maxCacheSize
+
+    # GQL cache-control max-age settings (in seconds)
+    maxAge = 15
+    timeTravelMaxAge = 86400 # 1 day
+
+[metrics]
+  host = "0.0.0.0"
+  port = 9002
+  [metrics.gql]
+    port = 9003
+
+[database]
+  type = "postgres"
+  host = "uniswap-watcher-db"
+  port = 5432
+  database = "uni-info-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+  maxQueryExecutionTime = 100
+
+  [database.extra]
+    # maximum number of clients the pool should contain
+    max = 20
+
+[upstream]
+  [upstream.ethServer]
+    gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql"
+    rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082"
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+  [upstream.uniWatcher]
+    gqlEndpoint = "http://uni-watcher-server:3003/graphql"
+    gqlSubscriptionEndpoint = "ws://uni-watcher-server:3003/graphql"
+
+  [upstream.tokenWatcher]
+    gqlEndpoint = "http://erc20-watcher-server:3001/graphql"
+    gqlSubscriptionEndpoint = "ws://erc20-watcher-server:3001/graphql"
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@uniswap-watcher-db:5432/uni-info-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 1000
+  eventsInBatch = 50
+  subgraphEventsOrder = true
+  blockDelayInMilliSecs = 2000
+  prefetchBlocksInMem = true
+  prefetchBlockCount = 10

config/watcher-uniswap-v3/uni-watcher.toml

@@ -0,0 +1,41 @@
+[server]
+  host = "0.0.0.0"
+  port = 3003
+  kind = "active"
+
+[metrics]
+  host = "0.0.0.0"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "uniswap-watcher-db"
+  port = 5432
+  database = "uni-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+  maxQueryExecutionTime = 100
+
+[upstream]
+  [upstream.ethServer]
+    gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql"
+    rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082"
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@uniswap-watcher-db:5432/uni-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 0
+  eventsInBatch = 50
+  lazyUpdateBlockProgress = true
+  blockDelayInMilliSecs = 2000
+  prefetchBlocksInMem = true
+  prefetchBlockCount = 10

config/watcher-uniswap-v3/watch-contract.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+set -u
+
+echo "Watching factory contract 0x1F98431c8aD98523631AE4a59f267346ea31F984"
+yarn watch:contract --address 0x1F98431c8aD98523631AE4a59f267346ea31F984 --kind factory --startingBlock 12369621 --checkpoint
+
+echo "Watching nfpm contract 0xC36442b4a4522E871399CD717aBDD847Ab11FE88"
+yarn watch:contract --address 0xC36442b4a4522E871399CD717aBDD847Ab11FE88 --kind nfpm --startingBlock 12369651 --checkpoint

container-build/cerc-uniswap-v3-info/Dockerfile

@@ -0,0 +1,13 @@
+FROM node:15.3.0-alpine3.10
+
+RUN apk --update --no-cache add make git
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Building uniswap-v3-info" && \
+    git checkout v0.1.1 && \
+    yarn
+
+CMD ["sh", "-c", "yarn start"]

container-build/cerc-uniswap-v3-info/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Build cerc/uniswap-v3-info
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/uniswap-v3-info:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/uniswap-v3-info

container-build/cerc-watcher-uniswap-v3/Dockerfile

@@ -0,0 +1,11 @@
+FROM node:16.17.1-alpine3.16
+
+RUN apk --update --no-cache add git python3 alpine-sdk
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Building uniswap-watcher-ts" && \
+    git checkout v0.3.4 && \
+    yarn && yarn build && yarn build:contracts

container-build/cerc-watcher-uniswap-v3/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Build cerc/watcher-uniswap-v3
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/watcher-uniswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/uniswap-watcher-ts

stacks/erc20/README.md

@@ -44,7 +44,7 @@ Instructions to deploy a local ERC20 watcher stack (core + watcher) for demonstr
   $ export TOKEN_ADDRESS=<TOKEN_ADDRESS>
   ```
 
-* Open `http://localhost:3001/graphql` (GraphQL Playground) in a browser window
+* Open `http://localhost:3002/graphql` (GraphQL Playground) in a browser window
 
 * Connect MetaMask to `http://localhost:8545` (with chain ID `99`)
 

stacks/uniswap-v3/README.md

@@ -0,0 +1,83 @@
+# Uniswap v3
+
+Instructions to deploy Uniswap v3 watcher stack (watcher + uniswap-v3-info frontend app) using [laconic-stack-orchestrator](../../README.md#setup)
+
+## Prerequisites
+
+* Access to [uniswap-watcher-ts](https://github.com/vulcanize/uniswap-watcher-ts).
+
+* This deployment expects core services to be running; specifically, it requires `ipld-eth-server` RPC and GQL endpoints. Update the `upstream.ethServer` endpoints in the [watcher config files](../../config/watcher-uniswap-v3) accordingly:
+
+  ```toml
+  [upstream]
+    [upstream.ethServer]
+      gqlApiEndpoint = "http://ipld-eth-server.example.com:8083/graphql"
+      rpcProviderEndpoint = "http://ipld-eth-server.example.com:8082"
+  ```
+
+* `uni-watcher` and `uni-info-watcher` database dumps (optional).
+
+## Setup
+
+* Clone / pull required repositories:
+
+  ```bash
+  $ laconic-so setup-repositories --include vulcanize/uniswap-watcher-ts,vulcanize/uniswap-v3-info --git-ssh --pull
+  ```
+
+* Build watcher and info app container images:
+
+  ```bash
+  $ laconic-so build-containers --include cerc/watcher-uniswap-v3,cerc/uniswap-v3-info
+  ```
+
+  This should create the required docker images in the local image registry.
+
+## Deploy
+
+* (Optional) Initialize the watcher database with existing database dumps if available:
+
+  * Start the watcher database to be initialized:
+
+    ```bash
+    $ laconic-so deploy-system --include watcher-uniswap-v3 up uniswap-watcher-db
+    ```
+
+  * Find the watcher database container's id using `docker ps` and export it for further usage:
+
+    ```bash
+    $ export CONTAINER_ID=<CONTAINER_ID>
+    ```
+
+  * Load watcher database dumps:
+
+    ```bash
+    # uni-watcher database
+    $ docker exec -i $CONTAINER_ID psql -U vdbm uni-watcher < UNI_WATCHER_DB_DUMP_FILE_PATH.sql
+
+    # uni-info-watcher database
+    $ docker exec -i $CONTAINER_ID psql -U vdbm uni-info-watcher < UNI_INFO_WATCHER_DB_DUMP_FILE_PATH.sql
+    ```
+
+* Start all the watcher and info app services:
+
+  ```bash
+  $ laconic-so deploy-system --include watcher-uniswap-v3 up
+  ```
+
+* Check that all the services are up and healthy:
+
+  ```bash
+  $ docker ps
+  ```
+
+  * The `uni-info-watcher` GraphQL Playground can be accessed at `http://localhost:3004/graphql`
+  * The frontend app can be accessed at `http://localhost:3006`
+
+## Clean up
+
+* To stop all the services running in background:
+
+  ```bash
+  $ laconic-so deploy-system --include watcher-uniswap-v3 down
+  ```

stacks/uniswap-v3/stack.yml

@@ -0,0 +1,10 @@
+version: "1.0"
+name: uniswap-v3
+repos:
+  - vulcanize/uniswap-watcher-ts
+  - vulcanize/uniswap-v3-info
+containers:
+  - cerc/watcher-uniswap-v3
+  - cerc/uniswap-v3-info
+pods:
+  - watcher-uniswap-v3