Merge pull request #45 from cerc-io/dboreham/run-mobymask-watcher

Run mobymask watcher

app/deploy_system.py

@@ -27,9 +27,10 @@ from .util import include_exclude_check
 @click.option("--include", help="only start these components")
 @click.option("--exclude", help="don\'t start these components")
 @click.option("--cluster", help="specify a non-default cluster name")
-@click.argument('command')  # help: command: up|down|ps
+@click.argument('command', required=True)  # help: command: up|down|ps
+@click.argument('services', nargs=-1)  # help: command: up|down|ps <service1> <service2>
 @click.pass_context
-def command(ctx, include, exclude, cluster, command):
+def command(ctx, include, exclude, cluster, command, services):
     '''deploy a stack'''
 
     # TODO: implement option exclusion and command value constraint lost with the move from argparse to click
@@ -70,11 +71,13 @@ def command(ctx, include, exclude, cluster, command):
     # See: https://gabrieldemarmiesse.github.io/python-on-whales/sub-commands/compose/
     docker = DockerClient(compose_files=compose_files, compose_project_name=cluster)
 
+    services_list = list(services) or None 
+ 
     if not dry_run:
         if command == "up":
             if verbose:
-                print("Running compose up")
+                print(f"Running compose up for services: {services_list}")
-            docker.compose.up(detach=True)
+            docker.compose.up(detach=True, services=services_list)
         elif command == "down":
             if verbose:
                 print("Running compose down")

compose/docker-compose-watcher-mobymask.yml

@@ -1,5 +1,7 @@
 version: '3.2'
 
+# TODO: remove hard-wired host ports
+
 services:
 
   watcher-db:
@@ -11,7 +13,7 @@ services:
       - POSTGRES_EXTENSION=mobymask-watcher-job-queue:pgcrypto
       - POSTGRES_PASSWORD=password
     volumes:
-      - ../common/initdb.d/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
       - watcher_db_data:/var/lib/postgresql/data
     ports:
       - "0.0.0.0:15432:5432"
@@ -30,10 +32,12 @@ services:
     image: cerc/watcher-mobymask:local
     command: ["sh", "-c", "yarn server"]
     volumes:
-      - ../common/watcher-ts/mobymask-watcher.toml:/app/packages/mobymask-watcher/environments/local.toml
+      - ../config/watcher-mobymask/mobymask-watcher.toml:/app/watcher-ts/packages/mobymask-watcher/environments/local.toml
     ports:
       - "0.0.0.0:3001:3001"
       - "0.0.0.0:9001:9001"
+    extra_hosts:
+      - "ipld-eth-server:host-gateway"
     healthcheck:
       test: ["CMD", "nc", "-v", "localhost", "3001"]
       interval: 20s
@@ -51,9 +55,11 @@ services:
     image: cerc/watcher-mobymask:local
     command: ["sh", "-c", "yarn job-runner"]
     volumes:
-      - ../common/watcher-ts/mobymask-watcher.toml:/app/packages/mobymask-watcher/environments/local.toml
+      - ../config/watcher-mobymask/mobymask-watcher.toml:/app/watcher-ts/packages/mobymask-watcher/environments/local.toml
     ports:
       - "0.0.0.0:9000:9000"
+    extra_hosts:
+      - "ipld-eth-server:host-gateway"
 
 volumes:
   indexer_db_data:

config/postgresql/multiple-postgressql-databases.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -e
+set -u
+
+function create_user_and_database() {
+	local database=$1
+	echo "  Creating user and database '$database'"
+	psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
+	    CREATE DATABASE "$database";
+	    GRANT ALL PRIVILEGES ON DATABASE "$database" TO $POSTGRES_USER;
+EOSQL
+}
+
+function create_extension() {
+	local database=$(echo $1 | tr ':' ' ' | awk  '{print $1}')
+	local extension=$(echo $1 | tr ':' ' ' | awk  '{print $2}')
+	echo "  Creating database '$database' extension '$extension'"
+	psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" "$database" <<-EOSQL
+	    CREATE EXTENSION "$extension";
+EOSQL
+}
+
+if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then
+	echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES"
+	for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do
+		create_user_and_database $db
+	done
+	echo "Multiple databases created"
+fi
+
+if [ -n "$POSTGRES_EXTENSION" ]; then
+	echo "Extension database creation requested: $POSTGRES_EXTENSION"
+	for db in $(echo $POSTGRES_EXTENSION | tr ',' ' '); do
+		create_extension $db
+	done
+	echo "Extensions created"
+fi

config/watcher-mobymask/mobymask-watcher.toml

@@ -0,0 +1,53 @@
+[server]
+  host = "0.0.0.0"
+  port = 3001
+  kind = "active"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # IPFS API address (can be taken from the output on running the IPFS daemon).
+  # ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001"
+
+  # Boolean to filter logs by contract.
+  filterLogs = true
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = -1
+
+[metrics]
+  host = "0.0.0.0"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "watcher-db"
+  port = 5432
+  database = "mobymask-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+
+[upstream]
+  [upstream.ethServer]
+    gqlApiEndpoint = "http://ipld-eth-server:8083/graphql"
+    rpcProviderEndpoint = "http://ipld-eth-server:8082"
+    blockDelayInMilliSecs = 60000
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@watcher-db/mobymask-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50

pod-list.txt

@@ -10,4 +10,5 @@ prometheus-grafana
 laconicd
 fixturenet-laconicd
 fixturenet-eth
+watcher-mobymask
 test

stacks/mobymask/README.md

@@ -1,12 +1,14 @@
 # Mobymask
 
-## Set up a Mobymask Watcher
+## Deploy the Mobymask Watcher
+The instructions below show how to deploy a mobymask watcher using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator#install)).
 
+This deployment expects that ipld-eth-server's endpoints are available on the local machine at http://ipld-eth-server.example.com:8083/graphql and http://ipld-eth-server.example.com:8082. More advanced configurations are supported by modifying the watcher's [config file](../../config/watcher-mobymask/mobymask-watcher.toml).
 ## Clone required repositories
 ```
 $ laconic-so setup-repositories
 ```
-Checkout required branches:
+Checkout required branches for the current release:
 ```
 $ cd ~/cerc/assemblyscript
 $ git checkout ng-integrate-asyncify
@@ -17,7 +19,18 @@ $ git checkout v0.2.13
 ```
 $ laconic-sh build-containers --include cerc/watcher-mobymask
 ```
+This should create a container with tag `cerc/watcher-mobymask` in the local image registry.
 ## Deploy the stack
+First the watcher database has to be initialized. Start only the watcher-db service:
+```
+$ laconic-so deploy-system --include watcher-mobymask up watcher-db
+```
+Next find the container's id using `docker ps` then run the following command to initialize the database:
+```
+$ docker exec -i <watcher-db-container> psql -U vdbm mobymask-watcher < config/watcher-mobymask/mobymask-watcher-db.sql
+```
+Finally start the remaining containers:
 ```
 $ laconic-sh deploy-system --include watcher-mobymask
 ```
+Correct operation should be verified by following the instructions [here](https://github.com/cerc-io/mobymask-watcher/tree/main/mainnet-watcher-only#run), checking GraphQL queries return valid results in the watcher's [playground](http://127.0.0.1:3001/graphql).

stacks/mobymask/stack.yml

@@ -1,3 +1,4 @@
+version: "1.0"
 name: mobymask-watcher
 repos:
   - cerc-io/watcher-ts/v0.2.13