Separate out watcher and web-apps in mobymask-v2 stack (#287)

* Separate out watcher and web-apps in mobymask stack

* Take L2 RPC endpoint from the env file

* Changes to run watcher and mobymask web-app separately

* Support running watcher without contract deployment and L2 txs

* Remove duplicate mobymask params env

* Add code comments

* Add instructions for running web-apps separately

* Self review fixes

* Fix timeout for mobymask-app on watcher server

---------

Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com>
Former-commit-id: 6f781ae303

app/data/compose/docker-compose-mobymask-app.yml

@@ -0,0 +1,31 @@
+version: '3.2'
+
+services:
+  # Builds and serves the MobyMask react-app
+  mobymask-app:
+    image: cerc/mobymask-ui:local
+    env_file:
+      - ../config/watcher-mobymask-v2/mobymask-params.env
+    # Waits for watcher server to be up before app build
+    # Required when running with watcher stack to get deployed contract address
+    command:
+      - sh
+      - -c
+      - ./wait-for-it.sh -h $${WATCHER_HOST} -p $${WATCHER_PORT} -s -t 0 -- ./mobymask-app-start.sh
+    volumes:
+      - ../config/wait-for-it.sh:/app/wait-for-it.sh
+      - ../config/watcher-mobymask-v2/mobymask-app-config.json:/app/src/mobymask-app-config.json
+      - ../config/watcher-mobymask-v2/mobymask-app-start.sh:/app/mobymask-app-start.sh
+      - mobymask_deployment:/server
+    ports:
+      - "0.0.0.0:3002:3000"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+    shm_size: '1GB'
+
+volumes:
+  mobymask_deployment:

app/data/compose/docker-compose-peer-test-app.yml

@@ -0,0 +1,21 @@
+version: '3.2'
+
+services:
+  peer-test-app:
+    # Builds and serves the peer-test react-app
+    image: cerc/react-peer:local
+    working_dir: /app/packages/test-app
+    env_file:
+      - ../config/watcher-mobymask-v2/mobymask-params.env
+    command: ["sh", "./test-app-start.sh"]
+    volumes:
+      - ../config/watcher-mobymask-v2/test-app-config.json:/app/packages/test-app/src/test-app-config.json
+      - ../config/watcher-mobymask-v2/test-app-start.sh:/app/packages/test-app/test-app-start.sh
+    ports:
+      - "0.0.0.0:3003:3000"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s

app/data/compose/docker-compose-watcher-mobymask-v2.yml

@@ -1,6 +1,7 @@
 version: '3.2'
 
 services:
+  # Starts the PostgreSQL database for watcher
   mobymask-watcher-db:
     restart: unless-stopped
     image: postgres:14-alpine
@@ -21,13 +22,17 @@ services:
       retries: 15
       start_period: 10s
 
+  # Deploys MobyMask contract and generates an invite link
+  # Deployment is skipped if DEPLOYED_CONTRACT env is already set
   mobymask:
     image: cerc/mobymask:local
     working_dir: /app/packages/server
     env_file:
       - ../config/watcher-mobymask-v2/optimism-params.env
+      - ../config/watcher-mobymask-v2/mobymask-params.env
     environment:
       - ENV=PROD
+    # Waits for L2 Optimism Geth and Node servers to be up before deploying contract
     command:
       - sh
       - -c
@@ -44,6 +49,7 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  # Starts the mobymask-v2-watcher server
   mobymask-watcher-server:
     restart: unless-stopped
     depends_on:
@@ -54,6 +60,7 @@ services:
     image: cerc/watcher-mobymask-v2:local
     env_file:
       - ../config/watcher-mobymask-v2/optimism-params.env
+      - ../config/watcher-mobymask-v2/mobymask-params.env
     command: ["sh", "start-server.sh"]
     volumes:
       - ../config/watcher-mobymask-v2/watcher-config-template.toml:/app/packages/mobymask-v2-watcher/environments/watcher-config-template.toml
@@ -63,6 +70,7 @@ services:
       - ../config/watcher-mobymask-v2/start-server.sh:/app/packages/mobymask-v2-watcher/start-server.sh
       - mobymask_deployment:/server
       - fixturenet_geth_accounts:/geth-accounts:ro
+    # Expose GQL, metrics and relay node ports
     ports:
       - "0.0.0.0:3001:3001"
       - "0.0.0.0:9001:9001"
@@ -76,48 +84,6 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
-  # TODO: Move to a separate pod
-  mobymask-app:
-    depends_on:
-      mobymask-watcher-server:
-        condition: service_healthy
-      mobymask:
-        condition: service_completed_successfully
-    image: cerc/mobymask-ui:local
-    command: ["sh", "mobymask-app-start.sh"]
-    volumes:
-      - ../config/watcher-mobymask-v2/mobymask-app.env:/app/.env
-      - ../config/watcher-mobymask-v2/mobymask-app-config.json:/app/src/mobymask-app-config.json
-      - ../config/watcher-mobymask-v2/mobymask-app-start.sh:/app/mobymask-app-start.sh
-      - mobymask_deployment:/server
-    ports:
-      - "0.0.0.0:3002:3000"
-    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3000"]
-      interval: 20s
-      timeout: 5s
-      retries: 15
-      start_period: 10s
-    shm_size: '1GB'
-
-  peer-test-app:
-    depends_on:
-      mobymask-watcher-server:
-        condition: service_healthy
-    image: cerc/react-peer:local
-    working_dir: /app/packages/test-app
-    command: ["sh", "-c", "yarn build && serve -s build"]
-    volumes:
-      - ../config/watcher-mobymask-v2/test-app-config.json:/app/packages/test-app/src/config.json
-    ports:
-      - "0.0.0.0:3003:3000"
-    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3000"]
-      interval: 20s
-      timeout: 5s
-      retries: 15
-      start_period: 10s
-
 volumes:
   mobymask_watcher_db_data:
   mobymask_deployment:

app/data/config/watcher-mobymask-v2/deploy-and-generate-invite.sh

@@ -4,6 +4,8 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
+echo "Using L2 RPC endpoint ${L2_GETH_RPC}"
+
 if [ -f /geth-accounts/accounts.csv ]; then
   echo "Using L1 private key from the mounted volume"
   # Read the private key of L1 account to deploy contract
@@ -16,10 +18,16 @@ fi
 jq --arg privateKey "$PRIVATE_KEY_DEPLOYER" '.privateKey = $privateKey' secrets-template.json > secrets.json
 
 # Set the RPC URL
-export L2_GETH_URL="http://${L2_GETH_HOST}:${L2_GETH_PORT}"
+jq --arg rpcUrl "$L2_GETH_RPC" '.rpcUrl = $rpcUrl' secrets.json > secrets_updated.json && mv secrets_updated.json secrets.json
-jq --arg rpcUrl "$L2_GETH_URL" '.rpcUrl = $rpcUrl' secrets.json > secrets_updated.json && mv secrets_updated.json secrets.json
 
-export RPC_URL="${L2_GETH_URL}"
+export RPC_URL="${L2_GETH_RPC}"
+
+# Check if DEPLOYED_CONTRACT environment variable set to skip contract deployment
+if [[ -n "$DEPLOYED_CONTRACT" ]]; then
+  echo "DEPLOYED_CONTRACT is set to '$DEPLOYED_CONTRACT'"
+  echo "Exiting without deploying contract"
+  exit 0
+fi
 
 # Check and exit if a deployment already exists (on restarts)
 if [ -f ./config.json ]; then
@@ -39,10 +47,10 @@ if [ -f ./config.json ]; then
   fi
 fi
 
-# Wait until balance for deployer account is reflected
+# Wait until balance for deployer account is updated
 cd ../hardhat
 while true; do
-  ACCOUNT_BALANCE=$(yarn balance --network optimism $PRIVATE_KEY_DEPLOYER | grep ETH)
+  ACCOUNT_BALANCE=$(yarn balance --network optimism "$PRIVATE_KEY_DEPLOYER" | grep ETH)
 
   if [ "$ACCOUNT_BALANCE" != "0.0 ETH" ]; then
     echo "Account balance updated: $ACCOUNT_BALANCE"

app/data/config/watcher-mobymask-v2/mobymask-app-config.json

@@ -1,4 +1,5 @@
 {
+  "name": "MobyMask",
   "relayNodes": [
     "/ip4/127.0.0.1/tcp/9090/ws/p2p/12D3KooWSPCsVkHVyLQoCqhu2YRPvvM7o6r6NRYyLM5zeA6Uig5t"
   ],

app/data/config/watcher-mobymask-v2/mobymask-app-start.sh

@@ -4,9 +4,22 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
-# Merging config files to get deployed contract address
+# Use config from mounted volume if available (when running web-app along with watcher stack)
-jq -s '.[0] * .[1]' /app/src/mobymask-app-config.json /server/config.json > /app/src/config.json
+if [ -f /server/config.json ]; then
+  echo "Merging config for deployed contract from mounted volume"
+  # Merging config files to get deployed contract address
+  jq -s '.[0] * .[1]' /app/src/mobymask-app-config.json /server/config.json > /app/src/config.json
+else
+  echo "Setting deployed contract details from env"
 
-npm run build
+  # Set config values from environment variables
+  jq --arg address "$DEPLOYED_CONTRACT" \
+    --argjson chainId $CHAIN_ID \
+    --argjson relayNodes "$RELAY_NODES" \
+    '.address = $address | .chainId = $chainId | .relayNodes = $relayNodes' \
+    /app/src/mobymask-app-config.json > /app/src/config.json
+fi
+
+REACT_APP_WATCHER_URI="$APP_WATCHER_URL/graphql" npm run build
 
 serve -s build

app/data/config/watcher-mobymask-v2/mobymask-app.env

@@ -1 +0,0 @@
-REACT_APP_WATCHER_URI=http://localhost:3001/graphql

app/data/config/watcher-mobymask-v2/mobymask-params.env

@@ -0,0 +1,14 @@
+# Change if pointing web app to external watcher endpoint
+WATCHER_HOST="mobymask-watcher-server"
+WATCHER_PORT=3001
+APP_WATCHER_URL="http://localhost:3001"
+
+# Set deployed MobyMask contract address to avoid deploying contract in stack
+# mobymask-app will use this contract address in config if run separately
+DEPLOYED_CONTRACT=
+
+# Chain ID is used by mobymask web-app for txs
+CHAIN_ID=42069
+
+# Set relay nodes to be used by web-apps
+RELAY_NODES=["/ip4/127.0.0.1/tcp/9090/ws/p2p/12D3KooWSPCsVkHVyLQoCqhu2YRPvvM7o6r6NRYyLM5zeA6Uig5t"]

app/data/config/watcher-mobymask-v2/optimism-params.env

@@ -1,9 +1,10 @@
 # Change if pointing to an external optimism geth endpoint
 
 # L2 endpoints
-# TODO: Add another env for complete URL to handle https
+L2_GETH_RPC="http://op-geth:8545"
 L2_GETH_HOST="op-geth"
 L2_GETH_PORT=8545
+
 L2_NODE_HOST="op-node"
 L2_NODE_PORT=8547
 

app/data/config/watcher-mobymask-v2/start-server.sh

@@ -4,8 +4,15 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
-# Assign deployed contract address from server config
+echo "Using L2 RPC endpoint ${L2_GETH_RPC}"
-CONTRACT_ADDRESS=$(jq -r '.address' /server/config.json | tr -d '"')
+
+# Use contract address from environment variable or set from config.json in mounted volume
+if [ -n "$DEPLOYED_CONTRACT" ]; then
+  CONTRACT_ADDRESS="${DEPLOYED_CONTRACT}"
+else
+  # Assign deployed contract address from server config (created by mobymask container after deploying contract)
+  CONTRACT_ADDRESS=$(jq -r '.address' /server/config.json | tr -d '"')
+fi
 
 if [ -f /geth-accounts/accounts.csv ]; then
   echo "Using L1 private key from the mounted volume"
@@ -15,11 +22,20 @@ else
   echo "Using PRIVATE_KEY_PEER from env"
 fi
 
-sed "s/REPLACE_WITH_PRIVATE_KEY/${PRIVATE_KEY_PEER}/" environments/watcher-config-template.toml > environments/local.toml
+# Set private key and contract address for watcher peer txs to L2 only if PRIVATE_KEY_PEER variable is set
-sed -i "s/REPLACE_WITH_CONTRACT_ADDRESS/${CONTRACT_ADDRESS}/" environments/local.toml
+if [ -n "$PRIVATE_KEY_PEER" ]; then
+  # Read in config template TOML file and modify it
+  CONTENT=$(cat environments/watcher-config-template.toml)
+  NEW_CONTENT=$(echo "$CONTENT" | sed -E "/\[metrics\]/i \\\n\n      [server.p2p.peer.l2TxConfig]\n        privateKey = \"${PRIVATE_KEY_PEER}\"\n        contractAddress = \"${CONTRACT_ADDRESS}\"\n")
 
-export L2_GETH_URL="http://${L2_GETH_HOST}:${L2_GETH_PORT}"
+  # Write the modified content to a watcher config file
-sed -i 's|REPLACE_WITH_L2_GETH_URL|'"${L2_GETH_URL}"'|' environments/local.toml
+  echo "$NEW_CONTENT" > environments/local.toml
+
+  sed -i 's|REPLACE_WITH_L2_GETH_RPC_ENDPOINT|'"${L2_GETH_RPC}"'|' environments/local.toml
+else
+  # Copy template config to watcher config without setting params for peer L2 txs
+  cp environments/watcher-config-template.toml environments/local.toml
+fi
 
 echo 'yarn server'
 yarn server

app/data/config/watcher-mobymask-v2/test-app-start.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# Set relay nodes in config from RELAY_NODES environment variable
+jq --argjson relayNodes "$RELAY_NODES" \
+  '.relayNodes = $relayNodes' \
+  ./src/test-app-config.json > ./src/config.json
+
+yarn build
+
+serve -s build

app/data/config/watcher-mobymask-v2/watcher-config-template.toml

@@ -36,10 +36,6 @@
       peerIdFile = './peer-id.json'
       enableDebugInfo = true
 
-      [server.p2p.peer.l2TxConfig]
-        privateKey = 'REPLACE_WITH_PRIVATE_KEY'
-        contractAddress = 'REPLACE_WITH_CONTRACT_ADDRESS'
-
 [metrics]
   host = "0.0.0.0"
   port = 9000
@@ -59,7 +55,7 @@
 [upstream]
   [upstream.ethServer]
     gqlApiEndpoint = "http://ipld-eth-server:8083/graphql"
-    rpcProviderEndpoint = "REPLACE_WITH_L2_GETH_URL"
+    rpcProviderEndpoint = "REPLACE_WITH_L2_GETH_RPC_ENDPOINT"
     blockDelayInMilliSecs = 60000
 
   [upstream.cache]

app/data/container-build/cerc-mobymask-ui/Dockerfile

@@ -1,6 +1,6 @@
 FROM node:18.15.0-alpine3.16
 
-RUN apk --update --no-cache add make git jq
+RUN apk --update --no-cache add make git jq bash
 
 WORKDIR /app
 

app/data/container-build/cerc-react-peer/Dockerfile

@@ -1,6 +1,6 @@
 FROM node:18.15.0-alpine3.16
 
-RUN apk --update --no-cache add make git python3
+RUN apk --update --no-cache add make git python3 jq
 
 WORKDIR /app
 

app/data/stacks/mobymask-v2/mobymask-only.md

@@ -4,30 +4,21 @@ Instructions to setup and deploy MobyMask v2 watcher independently
 
 ## Setup
 
-Prerequisite: An L2 Optimism RPC endpoint
+Prerequisite: L2 Optimism Geth and Node RPC endpoints
 
 Clone required repositories:
 
 ```bash
-laconic-so --stack mobymask-v2 setup-repositories --include cerc-io/MobyMask,cerc-io/watcher-ts,cerc-io/react-peer,cerc-io/mobymask-ui
+laconic-so --stack mobymask-v2 setup-repositories --include cerc-io/MobyMask,cerc-io/watcher-ts
 ```
 
 Checkout to the required versions and branches in repos:
 
-```bash
 ```bash
 # watcher-ts
 cd ~/cerc/watcher-ts
 git checkout v0.2.34
 
-# react-peer
-cd ~/cerc/react-peer
-git checkout v0.2.31
-
-# mobymask-ui
-cd ~/cerc/mobymask-ui
-git checkout laconic
-
 # MobyMask
 cd ~/cerc/MobyMask
 git checkout v0.1.2
@@ -36,20 +27,24 @@ git checkout v0.1.2
 Build the container images:
 
 ```bash
-laconic-so --stack mobymask-v2 build-containers --include cerc/watcher-mobymask-v2,cerc/react-peer,cerc/mobymask-ui,cerc/mobymask
+laconic-so --stack mobymask-v2 build-containers --include cerc/watcher-mobymask-v2,cerc/mobymask
 ```
 
 This should create the required docker images in the local image registry
 
 ## Deploy
 
-Update the [optimism-params.env](../../config/watcher-mobymask-v2/optimism-params.env) file with Optimism endpoints and other params if running Optimism separately
+### Configuration
 
+* In [mobymask-params.env](../../config/watcher-mobymask-v2/mobymask-params.env) file set `DEPLOYED_CONTRACT` to existing deployed mobymask contract address
+  * Setting `DEPLOYED_CONTRACT` will skip contract deployment when running stack
+* Update the [optimism-params.env](../../config/watcher-mobymask-v2/optimism-params.env) file with Optimism endpoints and other params for the Optimism running separately
+  * If `PRIVATE_KEY_PEER` is not set the inline watcher peer will not send txs to L2 on receiving P2P network messages
 * NOTE:
   * Stack Orchestrator needs to be run in [`dev`](/docs/CONTRIBUTING.md#install-developer-mode) mode to be able to edit the env file
   * If Optimism is running on the host machine, use `host.docker.internal` as the hostname to access the host port
 
-Deploy the stack:
+### Deploy the stack
 
 ```bash
 laconic-so --stack mobymask-v2 deploy --include watcher-mobymask-v2 up
@@ -67,7 +62,9 @@ docker ps
 docker logs -f <CONTAINER_ID>
 ```
 
-See [Tests](./README.md#tests) and [Demo](./README.md#demo) to interact with stack
+## Tests
+
+See [Tests](./README.md#tests)
 
 ## Clean up
 

app/data/stacks/mobymask-v2/stack.yml

@@ -27,3 +27,5 @@ pods:
   - fixturenet-eth
   - fixturenet-optimism
   - watcher-mobymask-v2
+  - mobymask-app
+  - peer-test-app

app/data/stacks/mobymask-v2/web-apps.md

@@ -0,0 +1,91 @@
+# Web Apps
+
+Instructions to setup and deploy MobyMask and Peer Test web apps
+
+## Setup
+
+Prerequisite: Watcher with GQL and relay node endpoints
+
+Clone required repositories:
+
+```bash
+laconic-so --stack mobymask-v2 setup-repositories --include cerc-io/react-peer,cerc-io/mobymask-ui
+```
+
+Checkout to the required versions and branches in repos:
+
+```bash
+# react-peer
+cd ~/cerc/react-peer
+git checkout v0.2.31
+
+# mobymask-ui
+cd ~/cerc/mobymask-ui
+git checkout laconic
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack mobymask-v2 build-containers --include cerc/react-peer-v2,cerc/mobymask-ui
+```
+
+This should create the required docker images in the local image registry
+
+## Deploy
+
+### Configuration
+
+* Update the [mobymask-params.env](../../config/watcher-mobymask-v2/mobymask-params.env) file with watcher endpoints and other params required by the web-apps
+  * `WATCHER_HOST` and `WATCHER_PORT` is used to check if watcher is up before building and deploying mobymask-app
+  * `APP_WATCHER_URL` is used by mobymask-app to make GQL queries
+  * `DEPLOYED_CONTRACT` and `CHAIN_ID` is used by mobymask-app in app config when creating messgaes for L2 txs
+  * `RELAY_NODES` is used by the web-apps to connect to the relay nodes (run in watcher)
+* NOTE:
+  * Stack Orchestrator needs to be run in [`dev`](/docs/CONTRIBUTING.md#install-developer-mode) mode to be able to edit the env file
+  * If watcher is running on the host machine, use `host.docker.internal` as the hostname to access the host port
+
+### Deploy the stack
+
+For running mobymask-app
+```bash
+laconic-so --stack mobymask-v2 deploy --include mobymask-app up
+```
+
+For running peer-test-app
+```bash
+laconic-so --stack mobymask-v2 deploy --include peer-test-app up
+```
+
+To list down and monitor the running containers:
+
+```bash
+docker ps
+
+# Check logs for a container
+docker logs -f <CONTAINER_ID>
+```
+
+## Clean up
+
+Stop all services running in the background:
+
+For mobymask-app
+```bash
+laconic-so --stack mobymask-v2 deploy --include mobymask-app down
+```
+
+For peer-test-app
+```bash
+laconic-so --stack mobymask-v2 deploy --include peer-test-app down
+```
+
+Clear volumes created by this stack:
+
+```bash
+# List all relevant volumes
+docker volume ls -q --filter "name=.*mobymask_deployment"
+
+# Remove all the listed volumes
+docker volume rm $(docker volume ls -q --filter "name=.*mobymask_deployment")
+```