From 5c7d4455001eb28a8881438730615409bb995574 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Tue, 16 May 2023 09:09:08 +0530 Subject: [PATCH] Add a stack for Gelato watcher (#394) * Add a stack for Gelato watcher * Add option to create and use a state snapshot * Add commands to create and import a state checkpoint * Rename ipld-eth-server endpoint env variables * Fix default env variable Former-commit-id: 8b4b5deba88bc5008a623a3826a5d2c0d76aadb6 --- .../compose/docker-compose-watcher-gelato.yml | 91 +++++++++++++++++ .../create-and-import-checkpoint.sh | 28 ++++++ .../config/watcher-gelato/start-job-runner.sh | 23 +++++ .../config/watcher-gelato/start-server.sh | 32 ++++++ .../watcher-config-template.toml | 75 ++++++++++++++ .../config/watcher-gelato/watcher-params.env | 13 +++ .../cerc-watcher-gelato/Dockerfile | 12 +++ .../cerc-watcher-gelato/build.sh | 9 ++ .../cerc-watcher-mobymask-v2/Dockerfile | 2 +- app/data/container-image-list.txt | 1 + app/data/pod-list.txt | 1 + app/data/repository-list.txt | 1 + app/data/stacks/azimuth/README.md | 4 +- app/data/stacks/gelato/README.md | 98 +++++++++++++++++++ app/data/stacks/gelato/stack.yml | 8 ++ 15 files changed, 395 insertions(+), 3 deletions(-) create mode 100644 app/data/compose/docker-compose-watcher-gelato.yml create mode 100755 app/data/config/watcher-gelato/create-and-import-checkpoint.sh create mode 100755 app/data/config/watcher-gelato/start-job-runner.sh create mode 100755 app/data/config/watcher-gelato/start-server.sh create mode 100644 app/data/config/watcher-gelato/watcher-config-template.toml create mode 100644 app/data/config/watcher-gelato/watcher-params.env create mode 100644 app/data/container-build/cerc-watcher-gelato/Dockerfile create mode 100755 app/data/container-build/cerc-watcher-gelato/build.sh create mode 100644 app/data/stacks/gelato/README.md create mode 100644 app/data/stacks/gelato/stack.yml diff --git a/app/data/compose/docker-compose-watcher-gelato.yml b/app/data/compose/docker-compose-watcher-gelato.yml new file mode 100644 index 00000000..fc607408 --- /dev/null +++ b/app/data/compose/docker-compose-watcher-gelato.yml @@ -0,0 +1,91 @@ +version: '3.2' + +services: + # Starts the PostgreSQL database for watcher + gelato-watcher-db: + restart: unless-stopped + image: postgres:14-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_MULTIPLE_DATABASES=gelato-watcher,gelato-watcher-job-queue + - POSTGRES_EXTENSION=gelato-watcher-job-queue:pgcrypto + - POSTGRES_PASSWORD=password + volumes: + - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh + - gelato_watcher_db_data:/var/lib/postgresql/data + ports: + - "0.0.0.0:15432:5432" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 10s + timeout: 5s + retries: 15 + start_period: 10s + + # Starts the gelato-watcher job runner + gelato-watcher-job-runner: + image: cerc/watcher-gelato:local + restart: unless-stopped + depends_on: + gelato-watcher-db: + condition: service_healthy + env_file: + - ../config/watcher-gelato/watcher-params.env + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + command: ["./start-job-runner.sh"] + volumes: + - ../config/watcher-gelato/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-gelato/start-job-runner.sh:/app/start-job-runner.sh + ports: + - "0.0.0.0:9000:9000" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "9000"] + interval: 10s + timeout: 5s + retries: 15 + start_period: 10s + extra_hosts: + - "host.docker.internal:host-gateway" + + # Starts the gelato-watcher server + gelato-watcher-server: + image: cerc/watcher-gelato:local + restart: unless-stopped + depends_on: + gelato-watcher-db: + condition: service_healthy + gelato-watcher-job-runner: + condition: service_healthy + env_file: + - ../config/watcher-gelato/watcher-params.env + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_USE_STATE_SNAPSHOT: ${CERC_USE_STATE_SNAPSHOT} + CERC_SNAPSHOT_GQL_ENDPOINT: ${CERC_SNAPSHOT_GQL_ENDPOINT} + CERC_SNAPSHOT_BLOCKHASH: ${CERC_SNAPSHOT_BLOCKHASH} + command: ["./start-server.sh"] + volumes: + - ../config/watcher-gelato/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-gelato/start-server.sh:/app/start-server.sh + - ../config/watcher-gelato/create-and-import-checkpoint.sh:/app/create-and-import-checkpoint.sh + - gelato_watcher_state_gql:/app/state_checkpoint + ports: + - "0.0.0.0:3008:3008" + - "0.0.0.0:9001:9001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3008"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + gelato_watcher_db_data: + gelato_watcher_state_gql: diff --git a/app/data/config/watcher-gelato/create-and-import-checkpoint.sh b/app/data/config/watcher-gelato/create-and-import-checkpoint.sh new file mode 100755 index 00000000..20b90503 --- /dev/null +++ b/app/data/config/watcher-gelato/create-and-import-checkpoint.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +CERC_SNAPSHOT_GQL_ENDPOINT="${CERC_SNAPSHOT_GQL_ENDPOINT:-${DEFAULT_CERC_SNAPSHOT_GQL_ENDPOINT}}" +CERC_SNAPSHOT_BLOCKHASH="${CERC_SNAPSHOT_BLOCKHASH:-${DEFAULT_CERC_SNAPSHOT_BLOCKHASH}}" + +CHECKPOINT_FILE_PATH="./state_checkpoint/state-gql-${CERC_SNAPSHOT_BLOCKHASH}" + +if [ -f "${CHECKPOINT_FILE_PATH}" ]; then + # Skip checkpoint creation if the file already exists + echo "File at ${CHECKPOINT_FILE_PATH} already exists, skipping checkpoint creation..." +else + # Create a checkpoint using GQL endpoint + echo "Creating a state checkpoint using GQL endpoint..." + yarn create-state-gql \ + --snapshot-block-hash "${CERC_SNAPSHOT_BLOCKHASH}" \ + --gql-endpoint "${CERC_SNAPSHOT_GQL_ENDPOINT}" \ + --output "${CHECKPOINT_FILE_PATH}" +fi + +echo "Initializing watcher using a state snapshot..." + +# Import the state checkpoint +# (skips if snapshot block is already indexed) +yarn import-state --import-file "${CHECKPOINT_FILE_PATH}" diff --git a/app/data/config/watcher-gelato/start-job-runner.sh b/app/data/config/watcher-gelato/start-job-runner.sh new file mode 100755 index 00000000..44f9e86f --- /dev/null +++ b/app/data/config/watcher-gelato/start-job-runner.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +CERC_IPLD_ETH_RPC="${CERC_IPLD_ETH_RPC:-${DEFAULT_CERC_IPLD_ETH_RPC}}" +CERC_IPLD_ETH_GQL="${CERC_IPLD_ETH_GQL:-${DEFAULT_CERC_IPLD_ETH_GQL}}" + +echo "Using ETH server RPC endpoint ${CERC_IPLD_ETH_RPC}" +echo "Using ETH server GQL endpoint ${CERC_IPLD_ETH_GQL}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \ + s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +echo "Running job-runner" +DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js diff --git a/app/data/config/watcher-gelato/start-server.sh b/app/data/config/watcher-gelato/start-server.sh new file mode 100755 index 00000000..4cd3cd73 --- /dev/null +++ b/app/data/config/watcher-gelato/start-server.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +CERC_IPLD_ETH_RPC="${CERC_IPLD_ETH_RPC:-${DEFAULT_CERC_IPLD_ETH_RPC}}" +CERC_IPLD_ETH_GQL="${CERC_IPLD_ETH_GQL:-${DEFAULT_CERC_IPLD_ETH_GQL}}" + +CERC_USE_STATE_SNAPSHOT="${CERC_USE_STATE_SNAPSHOT:-${DEFAULT_CERC_USE_STATE_SNAPSHOT}}" + +echo "Using ETH server RPC endpoint ${CERC_IPLD_ETH_RPC}" +echo "Using ETH server GQL endpoint ${CERC_IPLD_ETH_GQL}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \ + s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +if [ "$CERC_USE_STATE_SNAPSHOT" = true ] ; then + ./create-and-import-checkpoint.sh +else + echo "Initializing watcher using fill..." + yarn fill --start-block $DEFAULT_CERC_GELATO_START_BLOCK --end-block $DEFAULT_CERC_GELATO_START_BLOCK +fi + +echo "Running active server" +DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js diff --git a/app/data/config/watcher-gelato/watcher-config-template.toml b/app/data/config/watcher-gelato/watcher-config-template.toml new file mode 100644 index 00000000..19c3a2f1 --- /dev/null +++ b/app/data/config/watcher-gelato/watcher-config-template.toml @@ -0,0 +1,75 @@ +[server] + host = "0.0.0.0" + port = 3008 + kind = "active" + + # Checkpointing state. + checkpointing = true + + # Checkpoint interval in number of blocks. + checkpointInterval = 2000 + + # Enable state creation + # CAUTION: Disable only if state creation is not desired or can be filled subsequently + enableState = true + + subgraphPath = "./subgraph" + + # Interval to restart wasm instance periodically + wasmRestartBlocksInterval = 20 + + # Interval in number of blocks at which to clear entities cache. + clearEntitiesCacheInterval = 1000 + + # Boolean to filter logs by contract. + filterLogs = true + + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 + + # GQL cache settings + [server.gqlCache] + enabled = true + + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize + + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day + +[metrics] + host = "0.0.0.0" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "gelato-watcher-db" + port = 5432 + database = "gelato-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + +[upstream] + [upstream.ethServer] + gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL" + rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC" + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@gelato-watcher-db/gelato-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 100 + eventsInBatch = 50 + blockDelayInMilliSecs = 2000 + prefetchBlocksInMem = true + prefetchBlockCount = 10 diff --git a/app/data/config/watcher-gelato/watcher-params.env b/app/data/config/watcher-gelato/watcher-params.env new file mode 100644 index 00000000..e9bcad76 --- /dev/null +++ b/app/data/config/watcher-gelato/watcher-params.env @@ -0,0 +1,13 @@ +# ipld-eth-server endpoints +DEFAULT_CERC_IPLD_ETH_RPC="http://ipld-eth-server:8082" +DEFAULT_CERC_IPLD_ETH_GQL="http://ipld-eth-server:8083/graphql" + +# Gelato start block +DEFAULT_CERC_GELATO_START_BLOCK=11361987 + +# Whether to use a state snapshot to initialize the watcher +DEFAULT_CERC_USE_STATE_SNAPSHOT=false + +# State snapshot params +DEFAULT_CERC_SNAPSHOT_GQL_ENDPOINT= +DEFAULT_CERC_SNAPSHOT_BLOCKHASH= diff --git a/app/data/container-build/cerc-watcher-gelato/Dockerfile b/app/data/container-build/cerc-watcher-gelato/Dockerfile new file mode 100644 index 00000000..67a94bf4 --- /dev/null +++ b/app/data/container-build/cerc-watcher-gelato/Dockerfile @@ -0,0 +1,12 @@ +FROM node:18.15.0-alpine3.16 + +RUN apk --update --no-cache add git python3 alpine-sdk bash + +WORKDIR /app + +COPY . . + +RUN echo "Building gelato-watcher-ts" && \ + yarn && yarn build + +WORKDIR /app diff --git a/app/data/container-build/cerc-watcher-gelato/build.sh b/app/data/container-build/cerc-watcher-gelato/build.sh new file mode 100755 index 00000000..d27987e7 --- /dev/null +++ b/app/data/container-build/cerc-watcher-gelato/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Build cerc/watcher-gelato + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/watcher-gelato:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/gelato-watcher-ts diff --git a/app/data/container-build/cerc-watcher-mobymask-v2/Dockerfile b/app/data/container-build/cerc-watcher-mobymask-v2/Dockerfile index 6c100091..3e07eec9 100644 --- a/app/data/container-build/cerc-watcher-mobymask-v2/Dockerfile +++ b/app/data/container-build/cerc-watcher-mobymask-v2/Dockerfile @@ -15,6 +15,6 @@ WORKDIR /app COPY . . RUN echo "Building mobymask-v2-watcher-ts" && \ - yarn && yarn build + yarn && yarn build WORKDIR /app diff --git a/app/data/container-image-list.txt b/app/data/container-image-list.txt index f3d9985f..ab0a87b5 100644 --- a/app/data/container-image-list.txt +++ b/app/data/container-image-list.txt @@ -39,3 +39,4 @@ cerc/optimism-op-proposer cerc/pocket cerc/watcher-azimuth cerc/ipld-eth-state-snapshot +cerc/watcher-gelato diff --git a/app/data/pod-list.txt b/app/data/pod-list.txt index 34d66ba6..6c98c76c 100644 --- a/app/data/pod-list.txt +++ b/app/data/pod-list.txt @@ -26,3 +26,4 @@ foundry fixturenet-optimism fixturenet-pocket watcher-azimuth +watcher-gelato diff --git a/app/data/repository-list.txt b/app/data/repository-list.txt index 4002d559..4b257813 100644 --- a/app/data/repository-list.txt +++ b/app/data/repository-list.txt @@ -31,3 +31,4 @@ pokt-network/pocket-core pokt-network/pocket-core-deployments cerc-io/azimuth-watcher-ts cerc-io/ipld-eth-state-snapshot +cerc-io/gelato-watcher-ts diff --git a/app/data/stacks/azimuth/README.md b/app/data/stacks/azimuth/README.md index d3e377da..c4e1671f 100644 --- a/app/data/stacks/azimuth/README.md +++ b/app/data/stacks/azimuth/README.md @@ -4,7 +4,7 @@ Instructions to setup and deploy Azimuth Watcher stack ## Setup -Prerequisite: ipld-eth-server RPC and GQL endpoints +Prerequisite: `ipld-eth-server` RPC and GQL endpoints Clone required repositories: @@ -41,7 +41,7 @@ This should create the required docker images in the local image registry. CERC_IPLD_ETH_GQL= ``` -* NOTE: If ipld-eth-server is running on the host machine, use `host.docker.internal` as the hostname to access host ports +* NOTE: If `ipld-eth-server` is running on the host machine, use `host.docker.internal` as the hostname to access host ports ### Deploy the stack diff --git a/app/data/stacks/gelato/README.md b/app/data/stacks/gelato/README.md new file mode 100644 index 00000000..4baaee51 --- /dev/null +++ b/app/data/stacks/gelato/README.md @@ -0,0 +1,98 @@ +# Gelato watcher + +Instructions to setup and deploy Gelato watcher using [laconic-stack-orchestrator](/README.md#install) + +## Setup + +Prerequisite: `ipld-eth-server` RPC and GQL endpoints + +Clone required repositories: + +```bash +laconic-so --stack gelato setup-repositories + +# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command +``` + +Checkout to required version in the repo: + +```bash +# gelato-watcher-ts +cd ~/cerc/gelato-watcher-ts +git checkout v0.1.0 +``` + +Build the container images: + +```bash +laconic-so --stack gelato build-containers +``` + +This should create the required docker images in the local image registry. + +## Deploy + +### Configuration + +Create and update an env file to be used in the next step ([defaults](../../config/watcher-gelato/watcher-params.env)): + + ```bash + # External ipld-eth-server endpoints + CERC_IPLD_ETH_RPC= + CERC_IPLD_ETH_GQL= + + # Whether to use a state snapshot to initialize the watcher + CERC_USE_STATE_SNAPSHOT=false + + # State snapshot params + # Required if CERC_USE_STATE_SNAPSHOT is set to true + CERC_SNAPSHOT_GQL_ENDPOINT= + CERC_SNAPSHOT_BLOCKHASH= + ``` + +* NOTE: If `ipld-eth-server` is running on the host machine, use `host.docker.internal` as the hostname to access the host port(s) + +### Deploy the stack + +```bash +laconic-so --stack gelato deploy --env-file up +``` + +To list down and monitor the running containers: + +```bash +laconic-so --stack gelato deploy ps + +# With status +docker ps + +# Check logs for a container +docker logs -f +``` + +The stack runs an active watcher with following endpoints exposed on the host ports: +* `3008`: watcher endpoint +* `9000`: watcher metrics +* `9001`: watcher GQL metrics + +## Web Apps + +TODO + +## Clean up + +Stop all services running in the background: + +```bash +laconic-so --stack gelato deploy down +``` + +Clear volumes created by this stack: + +```bash +# List all relevant volumes +docker volume ls -q --filter "name=.*gelato_watcher_db_data|.*gelato_watcher_state_gql" + +# Remove all the listed volumes +docker volume rm $(docker volume ls -q --filter "name=.*gelato_watcher_db_data|.*gelato_watcher_state_gql") +``` diff --git a/app/data/stacks/gelato/stack.yml b/app/data/stacks/gelato/stack.yml new file mode 100644 index 00000000..51a8d64f --- /dev/null +++ b/app/data/stacks/gelato/stack.yml @@ -0,0 +1,8 @@ +version: "1.0" +name: gelato +repos: + - cerc-io/gelato-watcher-ts +containers: + - cerc/watcher-gelato +pods: + - watcher-gelato