diff --git a/app/data/compose/docker-compose-watcher-azimuth.yml b/app/data/compose/docker-compose-watcher-azimuth.yml new file mode 100644 index 00000000..327c77fc --- /dev/null +++ b/app/data/compose/docker-compose-watcher-azimuth.yml @@ -0,0 +1,304 @@ +version: '3.2' + +services: + # Starts the PostgreSQL database for watchers + watcher-db: + restart: unless-stopped + image: postgres:14-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_MULTIPLE_DATABASES=azimuth-watcher,azimuth-watcher-job-queue,censures-watcher,censures-watcher-job-queue,claims-watcher,claims-watcher-job-queue,conditional-star-release-watcher,conditional-star-release-watcher-job-queue,delegated-sending-watcher,delegated-sending-watcher-job-queue,ecliptic-watcher,ecliptic-watcher-job-queue,linear-star-release-watcher,linear-star-release-watcher-job-queue,polls-watcher,polls-watcher-job-queue + - POSTGRES_EXTENSION=azimuth-watcher-job-queue:pgcrypto,censures-watcher-job-queue:pgcrypto,claims-watcher-job-queue:pgcrypto,conditional-star-release-watcher-job-queue:pgcrypto,delegated-sending-watcher-job-queue:pgcrypto,ecliptic-watcher-job-queue:pgcrypto,linear-star-release-watcher-job-queue:pgcrypto,polls-watcher-job-queue:pgcrypto, + - POSTGRES_PASSWORD=password + volumes: + - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh + - watcher_db_data:/var/lib/postgresql/data + ports: + - "0.0.0.0:15432:5432" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + + # Starts the azimuth-watcher server + azimuth-watcher-server: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + env_file: + - ../config/watcher-azimuth/watcher-params.env + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + working_dir: /app/packages/azimuth-watcher + command: "./start-server.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/azimuth-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/azimuth-watcher/merge-toml.js + - ../config/watcher-azimuth/start-server.sh:/app/packages/azimuth-watcher/start-server.sh + ports: + - "3001" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "3001"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + # Starts the censures-watcher server + censures-watcher-server: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + env_file: + - ../config/watcher-azimuth/watcher-params.env + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + working_dir: /app/packages/censures-watcher + command: "./start-server.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/censures-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/censures-watcher/merge-toml.js + - ../config/watcher-azimuth/start-server.sh:/app/packages/censures-watcher/start-server.sh + ports: + - "3002" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "3002"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + # Starts the claims-watcher server + claims-watcher-server: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + env_file: + - ../config/watcher-azimuth/watcher-params.env + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + working_dir: /app/packages/claims-watcher + command: "./start-server.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/claims-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/claims-watcher/merge-toml.js + - ../config/watcher-azimuth/start-server.sh:/app/packages/claims-watcher/start-server.sh + ports: + - "3003" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "3003"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + # Starts the conditional-star-release-watcher server + conditional-star-release-watcher-server: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + env_file: + - ../config/watcher-azimuth/watcher-params.env + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + working_dir: /app/packages/conditional-star-release-watcher + command: "./start-server.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/conditional-star-release-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/conditional-star-release-watcher/merge-toml.js + - ../config/watcher-azimuth/start-server.sh:/app/packages/conditional-star-release-watcher/start-server.sh + ports: + - "3004" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "3004"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + # Starts the delegated-sending-watcher server + delegated-sending-watcher-server: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + env_file: + - ../config/watcher-azimuth/watcher-params.env + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + working_dir: /app/packages/delegated-sending-watcher + command: "./start-server.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/delegated-sending-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/delegated-sending-watcher/merge-toml.js + - ../config/watcher-azimuth/start-server.sh:/app/packages/delegated-sending-watcher/start-server.sh + ports: + - "3005" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "3005"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + # Starts the ecliptic-watcher server + ecliptic-watcher-server: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + env_file: + - ../config/watcher-azimuth/watcher-params.env + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + working_dir: /app/packages/ecliptic-watcher + command: "./start-server.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/ecliptic-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/ecliptic-watcher/merge-toml.js + - ../config/watcher-azimuth/start-server.sh:/app/packages/ecliptic-watcher/start-server.sh + ports: + - "3006" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "3006"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + # Starts the linear-star-release-watcher server + linear-star-release-watcher-server: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + env_file: + - ../config/watcher-azimuth/watcher-params.env + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + working_dir: /app/packages/linear-star-release-watcher + command: "./start-server.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/linear-star-release-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/linear-star-release-watcher/merge-toml.js + - ../config/watcher-azimuth/start-server.sh:/app/packages/linear-star-release-watcher/start-server.sh + ports: + - "3007" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "3007"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + # Starts the polls-watcher server + polls-watcher-server: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + env_file: + - ../config/watcher-azimuth/watcher-params.env + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + working_dir: /app/packages/polls-watcher + command: "./start-server.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/polls-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/polls-watcher/merge-toml.js + - ../config/watcher-azimuth/start-server.sh:/app/packages/polls-watcher/start-server.sh + ports: + - "3008" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "3008"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + # Starts the gateway-server for proxying queries + gateway-server: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + azimuth-watcher-server: + condition: service_healthy + censures-watcher-server: + condition: service_healthy + claims-watcher-server: + condition: service_healthy + conditional-star-release-watcher-server: + condition: service_healthy + delegated-sending-watcher-server: + condition: service_healthy + ecliptic-watcher-server: + condition: service_healthy + linear-star-release-watcher-server: + condition: service_healthy + polls-watcher-server: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + working_dir: /app/packages/gateway-server + command: "yarn server" + volumes: + - ../config/watcher-azimuth/gateway-watchers.json:/app/packages/gateway-server/dist/watchers.json + ports: + - "0.0.0.0:4000:4000" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "4000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + watcher_db_data: diff --git a/app/data/config/watcher-azimuth/gateway-watchers.json b/app/data/config/watcher-azimuth/gateway-watchers.json new file mode 100644 index 00000000..40687556 --- /dev/null +++ b/app/data/config/watcher-azimuth/gateway-watchers.json @@ -0,0 +1,34 @@ +[ + { + "endpoint": "http://azimuth-watcher-server:3001/graphql", + "prefix": "azimuth" + }, + { + "endpoint": "http://censures-watcher-server:3002/graphql", + "prefix": "censures" + }, + { + "endpoint": "http://claims-watcher-server:3003/graphql", + "prefix": "claims" + }, + { + "endpoint": "http://conditional-star-release-watcher-server:3004/graphql", + "prefix": "conditionalStarRelease" + }, + { + "endpoint": "http://delegated-sending-watcher-server:3005/graphql", + "prefix": "delegatedSending" + }, + { + "endpoint": "http://ecliptic-watcher-server:3006/graphql", + "prefix": "ecliptic" + }, + { + "endpoint": "http://linear-star-release-watcher-server:3007/graphql", + "prefix": "linearStarRelease" + }, + { + "endpoint": "http://polls-watcher-server:3008/graphql", + "prefix": "polls" + } +] diff --git a/app/data/config/watcher-azimuth/merge-toml.js b/app/data/config/watcher-azimuth/merge-toml.js new file mode 100644 index 00000000..9224baec --- /dev/null +++ b/app/data/config/watcher-azimuth/merge-toml.js @@ -0,0 +1,31 @@ +const fs = require('fs'); +const tomlJS = require('toml-js'); +const toml = require('toml'); +const { merge } = require('lodash') + +const main = () => { + const overrideConfigString = fs.readFileSync('environments/watcher-config.toml', 'utf-8'); + const configString = fs.readFileSync('environments/local.toml', 'utf-8'); + const overrideConfig = toml.parse(overrideConfigString) + const config = toml.parse(configString) + + // Merge configs + const updatedConfig = merge(config, overrideConfig); + + // Form dbConnectionString for jobQueue DB + const parts = config.jobQueue.dbConnectionString.split("://"); + const credsAndDB = parts[1].split("@"); + const creds = credsAndDB[0].split(":"); + creds[0] = overrideConfig.database.username; + creds[1] = overrideConfig.database.password; + credsAndDB[0] = creds.join(":"); + const dbName = credsAndDB[1].split("/")[1] + credsAndDB[1] = [overrideConfig.database.host, dbName].join("/"); + parts[1] = credsAndDB.join("@"); + + updatedConfig.jobQueue.dbConnectionString = parts.join("://"); + + fs.writeFileSync('environments/local.toml', tomlJS.dump(updatedConfig), 'utf-8'); +} + +main(); diff --git a/app/data/config/watcher-azimuth/start-server.sh b/app/data/config/watcher-azimuth/start-server.sh new file mode 100755 index 00000000..c84c58d0 --- /dev/null +++ b/app/data/config/watcher-azimuth/start-server.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +CERC_IPLD_ETH_RPC="${CERC_IPLD_ETH_RPC:-${DEFAULT_CERC_IPLD_ETH_RPC}}" +CERC_IPLD_ETH_GQL="${CERC_IPLD_ETH_GQL:-${DEFAULT_CERC_IPLD_ETH_GQL}}" + +echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}" +echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}" + +# Replace env variables in template TOML file +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \ + s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/watcher-config.toml + +# Merge SO watcher config with existing config file +node merge-toml.js + +echo 'yarn server' +yarn server diff --git a/app/data/config/watcher-azimuth/watcher-config-template.toml b/app/data/config/watcher-azimuth/watcher-config-template.toml new file mode 100644 index 00000000..1a4616fc --- /dev/null +++ b/app/data/config/watcher-azimuth/watcher-config-template.toml @@ -0,0 +1,14 @@ +[server] + host = "0.0.0.0" + maxSimultaneousRequests = -1 + +[database] + host = "watcher-db" + port = 5432 + username = "vdbm" + password = "password" + +[upstream] + [upstream.ethServer] + gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL" + rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC" diff --git a/app/data/config/watcher-azimuth/watcher-params.env b/app/data/config/watcher-azimuth/watcher-params.env new file mode 100644 index 00000000..8fcdc2d6 --- /dev/null +++ b/app/data/config/watcher-azimuth/watcher-params.env @@ -0,0 +1,5 @@ +# Defaults + +# ipld-eth-server endpoints +DEFAULT_CERC_IPLD_ETH_RPC= +DEFAULT_CERC_IPLD_ETH_GQL= diff --git a/app/data/container-build/cerc-watcher-azimuth/Dockerfile b/app/data/container-build/cerc-watcher-azimuth/Dockerfile new file mode 100644 index 00000000..eaf717aa --- /dev/null +++ b/app/data/container-build/cerc-watcher-azimuth/Dockerfile @@ -0,0 +1,13 @@ +FROM node:18.16.0-alpine3.16 + +RUN apk --update --no-cache add git python3 alpine-sdk + +WORKDIR /app + +COPY . . + +RUN echo "Building azimuth-watcher-ts" && \ + yarn && yarn build + +RUN echo "Install toml-js to update watcher config files" && \ + yarn add --dev --ignore-workspace-root-check toml-js diff --git a/app/data/container-build/cerc-watcher-azimuth/build.sh b/app/data/container-build/cerc-watcher-azimuth/build.sh new file mode 100755 index 00000000..522f985b --- /dev/null +++ b/app/data/container-build/cerc-watcher-azimuth/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Build cerc/watcher-azimuth + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/watcher-azimuth:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/azimuth-watcher-ts diff --git a/app/data/container-image-list.txt b/app/data/container-image-list.txt index 0fe2be5c..ba0eaf23 100644 --- a/app/data/container-image-list.txt +++ b/app/data/container-image-list.txt @@ -37,3 +37,4 @@ cerc/optimism-op-batcher cerc/optimism-op-node cerc/optimism-op-proposer cerc/pocket +cerc/watcher-azimuth diff --git a/app/data/pod-list.txt b/app/data/pod-list.txt index a3ae8f44..34d66ba6 100644 --- a/app/data/pod-list.txt +++ b/app/data/pod-list.txt @@ -25,3 +25,4 @@ kubo foundry fixturenet-optimism fixturenet-pocket +watcher-azimuth diff --git a/app/data/repository-list.txt b/app/data/repository-list.txt index 438b9bbb..7035de2a 100644 --- a/app/data/repository-list.txt +++ b/app/data/repository-list.txt @@ -29,3 +29,4 @@ ethereum-optimism/op-geth ethereum-optimism/optimism pokt-network/pocket-core pokt-network/pocket-core-deployments +cerc-io/azimuth-watcher-ts diff --git a/app/data/stacks/azimuth/README.md b/app/data/stacks/azimuth/README.md new file mode 100644 index 00000000..d3e377da --- /dev/null +++ b/app/data/stacks/azimuth/README.md @@ -0,0 +1,72 @@ +# Azimuth Watcher + +Instructions to setup and deploy Azimuth Watcher stack + +## Setup + +Prerequisite: ipld-eth-server RPC and GQL endpoints + +Clone required repositories: + +```bash +laconic-so --stack azimuth setup-repositories +``` + +NOTE: If the repository already exists and checked out to a different version, `setup-repositories` command will throw an error. +For getting around this, the `azimuth-watcher-ts` repository can be removed and then run the command. + +Checkout to the required versions and branches in repos + +```bash +# azimuth-watcher-ts +cd ~/cerc/azimuth-watcher-ts +git checkout v0.1.0 +``` + +Build the container images: + +```bash +laconic-so --stack azimuth build-containers +``` + +This should create the required docker images in the local image registry. + +### Configuration + +* Create and update an env file to be used in the next step: + + ```bash + # External ipld-eth-server endpoints + CERC_IPLD_ETH_RPC= + CERC_IPLD_ETH_GQL= + ``` + +* NOTE: If ipld-eth-server is running on the host machine, use `host.docker.internal` as the hostname to access host ports + +### Deploy the stack + +* Deploy the containers: + + ```bash + laconic-so --stack azimuth deploy-system --env-file up + ``` + +* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy` + +## Clean up + +Stop all the services running in background run: + +```bash +laconic-so --stack azimuth deploy-system down +``` + +Clear volumes created by this stack: + +```bash +# List all relevant volumes +docker volume ls -q --filter "name=.*watcher_db_data" + +# Remove all the listed volumes +docker volume rm $(docker volume ls -q --filter "name=.*watcher_db_data") +``` diff --git a/app/data/stacks/azimuth/stack.yml b/app/data/stacks/azimuth/stack.yml new file mode 100644 index 00000000..fc3c8473 --- /dev/null +++ b/app/data/stacks/azimuth/stack.yml @@ -0,0 +1,8 @@ +version: "1.0" +name: azimuth +repos: + - cerc-io/azimuth-watcher-ts +containers: + - cerc/watcher-azimuth +pods: + - watcher-azimuth