Add stack for azimuth watchers with gateway-server (#379)

* Setup gateway-server with watchers

* Add js script to merge toml config files

* Remove individual watcher configs

* Add all azimuth watchers in stack

* Fix toml-js install

* Use env variables for ipld-eth-server endpoints

* Checkout to version tag in azimuth-watcher-ts repo

Former-commit-id: 5a94aed7f7

app/data/compose/docker-compose-watcher-azimuth.yml

@@ -0,0 +1,304 @@
+version: '3.2'
+
+services:
+  # Starts the PostgreSQL database for watchers
+  watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=azimuth-watcher,azimuth-watcher-job-queue,censures-watcher,censures-watcher-job-queue,claims-watcher,claims-watcher-job-queue,conditional-star-release-watcher,conditional-star-release-watcher-job-queue,delegated-sending-watcher,delegated-sending-watcher-job-queue,ecliptic-watcher,ecliptic-watcher-job-queue,linear-star-release-watcher,linear-star-release-watcher-job-queue,polls-watcher,polls-watcher-job-queue
+      - POSTGRES_EXTENSION=azimuth-watcher-job-queue:pgcrypto,censures-watcher-job-queue:pgcrypto,claims-watcher-job-queue:pgcrypto,conditional-star-release-watcher-job-queue:pgcrypto,delegated-sending-watcher-job-queue:pgcrypto,ecliptic-watcher-job-queue:pgcrypto,linear-star-release-watcher-job-queue:pgcrypto,polls-watcher-job-queue:pgcrypto,
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "0.0.0.0:15432:5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  # Starts the azimuth-watcher server
+  azimuth-watcher-server:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    env_file:
+      - ../config/watcher-azimuth/watcher-params.env
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+    working_dir: /app/packages/azimuth-watcher
+    command: "./start-server.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/azimuth-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/azimuth-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-server.sh:/app/packages/azimuth-watcher/start-server.sh
+    ports:
+      - "3001"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "3001"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  # Starts the censures-watcher server
+  censures-watcher-server:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    env_file:
+      - ../config/watcher-azimuth/watcher-params.env
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+    working_dir: /app/packages/censures-watcher
+    command: "./start-server.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/censures-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/censures-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-server.sh:/app/packages/censures-watcher/start-server.sh
+    ports:
+      - "3002"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "3002"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  # Starts the claims-watcher server
+  claims-watcher-server:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    env_file:
+      - ../config/watcher-azimuth/watcher-params.env
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+    working_dir: /app/packages/claims-watcher
+    command: "./start-server.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/claims-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/claims-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-server.sh:/app/packages/claims-watcher/start-server.sh
+    ports:
+      - "3003"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "3003"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  # Starts the conditional-star-release-watcher server
+  conditional-star-release-watcher-server:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    env_file:
+      - ../config/watcher-azimuth/watcher-params.env
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+    working_dir: /app/packages/conditional-star-release-watcher
+    command: "./start-server.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/conditional-star-release-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/conditional-star-release-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-server.sh:/app/packages/conditional-star-release-watcher/start-server.sh
+    ports:
+      - "3004"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "3004"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  # Starts the delegated-sending-watcher server
+  delegated-sending-watcher-server:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    env_file:
+      - ../config/watcher-azimuth/watcher-params.env
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+    working_dir: /app/packages/delegated-sending-watcher
+    command: "./start-server.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/delegated-sending-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/delegated-sending-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-server.sh:/app/packages/delegated-sending-watcher/start-server.sh
+    ports:
+      - "3005"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "3005"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  # Starts the ecliptic-watcher server
+  ecliptic-watcher-server:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    env_file:
+      - ../config/watcher-azimuth/watcher-params.env
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+    working_dir: /app/packages/ecliptic-watcher
+    command: "./start-server.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/ecliptic-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/ecliptic-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-server.sh:/app/packages/ecliptic-watcher/start-server.sh
+    ports:
+      - "3006"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "3006"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  # Starts the linear-star-release-watcher server
+  linear-star-release-watcher-server:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    env_file:
+      - ../config/watcher-azimuth/watcher-params.env
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+    working_dir: /app/packages/linear-star-release-watcher
+    command: "./start-server.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/linear-star-release-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/linear-star-release-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-server.sh:/app/packages/linear-star-release-watcher/start-server.sh
+    ports:
+      - "3007"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "3007"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  # Starts the polls-watcher server
+  polls-watcher-server:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    env_file:
+      - ../config/watcher-azimuth/watcher-params.env
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+    working_dir: /app/packages/polls-watcher
+    command: "./start-server.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/polls-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/polls-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-server.sh:/app/packages/polls-watcher/start-server.sh
+    ports:
+      - "3008"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "3008"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  # Starts the gateway-server for proxying queries
+  gateway-server:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      azimuth-watcher-server:
+        condition: service_healthy
+      censures-watcher-server:
+        condition: service_healthy
+      claims-watcher-server:
+        condition: service_healthy
+      conditional-star-release-watcher-server:
+        condition: service_healthy
+      delegated-sending-watcher-server:
+        condition: service_healthy
+      ecliptic-watcher-server:
+        condition: service_healthy
+      linear-star-release-watcher-server:
+        condition: service_healthy
+      polls-watcher-server:
+        condition: service_healthy
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+    working_dir: /app/packages/gateway-server
+    command: "yarn server"
+    volumes:
+      - ../config/watcher-azimuth/gateway-watchers.json:/app/packages/gateway-server/dist/watchers.json
+    ports:
+      - "0.0.0.0:4000:4000"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "4000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  watcher_db_data:

app/data/config/watcher-azimuth/gateway-watchers.json

@@ -0,0 +1,34 @@
+[
+  {
+    "endpoint": "http://azimuth-watcher-server:3001/graphql",
+    "prefix": "azimuth"
+  },
+  {
+    "endpoint": "http://censures-watcher-server:3002/graphql",
+    "prefix": "censures"
+  },
+  {
+    "endpoint": "http://claims-watcher-server:3003/graphql",
+    "prefix": "claims"
+  },
+  {
+    "endpoint": "http://conditional-star-release-watcher-server:3004/graphql",
+    "prefix": "conditionalStarRelease"
+  },
+  {
+    "endpoint": "http://delegated-sending-watcher-server:3005/graphql",
+    "prefix": "delegatedSending"
+  },
+  {
+    "endpoint": "http://ecliptic-watcher-server:3006/graphql",
+    "prefix": "ecliptic"
+  },
+  {
+    "endpoint": "http://linear-star-release-watcher-server:3007/graphql",
+    "prefix": "linearStarRelease"
+  },
+  {
+    "endpoint": "http://polls-watcher-server:3008/graphql",
+    "prefix": "polls"
+  }
+]

app/data/config/watcher-azimuth/merge-toml.js

@@ -0,0 +1,31 @@
+const fs = require('fs');
+const tomlJS = require('toml-js');
+const toml = require('toml');
+const { merge } = require('lodash')
+
+const main = () => {
+  const overrideConfigString = fs.readFileSync('environments/watcher-config.toml', 'utf-8');
+  const configString = fs.readFileSync('environments/local.toml', 'utf-8');
+  const overrideConfig = toml.parse(overrideConfigString)
+  const config = toml.parse(configString)
+
+  // Merge configs
+  const updatedConfig = merge(config, overrideConfig);
+
+  // Form dbConnectionString for jobQueue DB
+  const parts = config.jobQueue.dbConnectionString.split("://");
+  const credsAndDB = parts[1].split("@");
+  const creds = credsAndDB[0].split(":");
+  creds[0] = overrideConfig.database.username;
+  creds[1] = overrideConfig.database.password;
+  credsAndDB[0] = creds.join(":");
+  const dbName = credsAndDB[1].split("/")[1]
+  credsAndDB[1] = [overrideConfig.database.host, dbName].join("/");
+  parts[1] = credsAndDB.join("@");
+
+  updatedConfig.jobQueue.dbConnectionString = parts.join("://");
+
+  fs.writeFileSync('environments/local.toml', tomlJS.dump(updatedConfig), 'utf-8');
+}
+
+main();

app/data/config/watcher-azimuth/start-server.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+CERC_IPLD_ETH_RPC="${CERC_IPLD_ETH_RPC:-${DEFAULT_CERC_IPLD_ETH_RPC}}"
+CERC_IPLD_ETH_GQL="${CERC_IPLD_ETH_GQL:-${DEFAULT_CERC_IPLD_ETH_GQL}}"
+
+echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
+echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
+
+# Replace env variables in template TOML file
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
+          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/watcher-config.toml
+
+# Merge SO watcher config with existing config file
+node merge-toml.js
+
+echo 'yarn server'
+yarn server

app/data/config/watcher-azimuth/watcher-config-template.toml

@@ -0,0 +1,14 @@
+[server]
+  host = "0.0.0.0"
+  maxSimultaneousRequests = -1
+
+[database]
+  host = "watcher-db"
+  port = 5432
+  username = "vdbm"
+  password = "password"
+
+[upstream]
+  [upstream.ethServer]
+    gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL"
+    rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC"

app/data/config/watcher-azimuth/watcher-params.env

@@ -0,0 +1,5 @@
+# Defaults
+
+# ipld-eth-server endpoints
+DEFAULT_CERC_IPLD_ETH_RPC=
+DEFAULT_CERC_IPLD_ETH_GQL=

app/data/container-build/cerc-watcher-azimuth/Dockerfile

@@ -0,0 +1,13 @@
+FROM node:18.16.0-alpine3.16
+
+RUN apk --update --no-cache add git python3 alpine-sdk
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Building azimuth-watcher-ts" && \
+    yarn && yarn build
+
+RUN echo "Install toml-js to update watcher config files" && \
+    yarn add --dev --ignore-workspace-root-check toml-js

app/data/container-build/cerc-watcher-azimuth/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Build cerc/watcher-azimuth
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/watcher-azimuth:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/azimuth-watcher-ts

app/data/container-image-list.txt

@@ -37,3 +37,4 @@ cerc/optimism-op-batcher
 cerc/optimism-op-node
 cerc/optimism-op-proposer
 cerc/pocket
+cerc/watcher-azimuth

app/data/pod-list.txt

@@ -25,3 +25,4 @@ kubo
 foundry
 fixturenet-optimism
 fixturenet-pocket
+watcher-azimuth

app/data/repository-list.txt

@@ -29,3 +29,4 @@ ethereum-optimism/op-geth
 ethereum-optimism/optimism
 pokt-network/pocket-core
 pokt-network/pocket-core-deployments
+cerc-io/azimuth-watcher-ts

app/data/stacks/azimuth/README.md

@@ -0,0 +1,72 @@
+# Azimuth Watcher
+
+Instructions to setup and deploy Azimuth Watcher stack
+
+## Setup
+
+Prerequisite: ipld-eth-server RPC and GQL endpoints
+
+Clone required repositories:
+
+```bash
+laconic-so --stack azimuth setup-repositories
+```
+
+NOTE: If the repository already exists and checked out to a different version, `setup-repositories` command will throw an error.
+For getting around this, the `azimuth-watcher-ts` repository can be removed and then run the command.
+
+Checkout to the required versions and branches in repos
+
+```bash
+# azimuth-watcher-ts
+cd ~/cerc/azimuth-watcher-ts
+git checkout v0.1.0
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack azimuth build-containers
+```
+
+This should create the required docker images in the local image registry.
+
+### Configuration
+
+* Create and update an env file to be used in the next step:
+
+  ```bash
+  # External ipld-eth-server endpoints
+  CERC_IPLD_ETH_RPC=
+  CERC_IPLD_ETH_GQL=
+  ```
+
+* NOTE: If ipld-eth-server is running on the host machine, use `host.docker.internal` as the hostname to access host ports
+
+### Deploy the stack
+
+* Deploy the containers:
+
+  ```bash
+  laconic-so --stack azimuth deploy-system --env-file <PATH_TO_ENV_FILE> up
+  ```
+
+* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy`
+
+## Clean up
+
+Stop all the services running in background run:
+
+```bash
+laconic-so --stack azimuth deploy-system down
+```
+
+Clear volumes created by this stack:
+
+```bash
+# List all relevant volumes
+docker volume ls -q --filter "name=.*watcher_db_data"
+
+# Remove all the listed volumes
+docker volume rm $(docker volume ls -q --filter "name=.*watcher_db_data")
+```

app/data/stacks/azimuth/stack.yml

@@ -0,0 +1,8 @@
+version: "1.0"
+name: azimuth
+repos:
+  - cerc-io/azimuth-watcher-ts
+containers:
+  - cerc/watcher-azimuth
+pods:
+  - watcher-azimuth