Merge pull request #45 from cerc-io/dboreham/run-mobymask-watcher

Run mobymask watcher
This commit is contained in:
David Boreham 2022-11-08 00:24:07 -07:00 committed by GitHub
commit f8d4bd9940
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 1187 additions and 10 deletions

View File

@ -27,9 +27,10 @@ from .util import include_exclude_check
@click.option("--include", help="only start these components") @click.option("--include", help="only start these components")
@click.option("--exclude", help="don\'t start these components") @click.option("--exclude", help="don\'t start these components")
@click.option("--cluster", help="specify a non-default cluster name") @click.option("--cluster", help="specify a non-default cluster name")
@click.argument('command') # help: command: up|down|ps @click.argument('command', required=True) # help: command: up|down|ps
@click.argument('services', nargs=-1) # help: command: up|down|ps <service1> <service2>
@click.pass_context @click.pass_context
def command(ctx, include, exclude, cluster, command): def command(ctx, include, exclude, cluster, command, services):
'''deploy a stack''' '''deploy a stack'''
# TODO: implement option exclusion and command value constraint lost with the move from argparse to click # TODO: implement option exclusion and command value constraint lost with the move from argparse to click
@ -70,11 +71,13 @@ def command(ctx, include, exclude, cluster, command):
# See: https://gabrieldemarmiesse.github.io/python-on-whales/sub-commands/compose/ # See: https://gabrieldemarmiesse.github.io/python-on-whales/sub-commands/compose/
docker = DockerClient(compose_files=compose_files, compose_project_name=cluster) docker = DockerClient(compose_files=compose_files, compose_project_name=cluster)
services_list = list(services) or None
if not dry_run: if not dry_run:
if command == "up": if command == "up":
if verbose: if verbose:
print("Running compose up") print(f"Running compose up for services: {services_list}")
docker.compose.up(detach=True) docker.compose.up(detach=True, services=services_list)
elif command == "down": elif command == "down":
if verbose: if verbose:
print("Running compose down") print("Running compose down")

View File

@ -1,5 +1,7 @@
version: '3.2' version: '3.2'
# TODO: remove hard-wired host ports
services: services:
watcher-db: watcher-db:
@ -11,7 +13,7 @@ services:
- POSTGRES_EXTENSION=mobymask-watcher-job-queue:pgcrypto - POSTGRES_EXTENSION=mobymask-watcher-job-queue:pgcrypto
- POSTGRES_PASSWORD=password - POSTGRES_PASSWORD=password
volumes: volumes:
- ../common/initdb.d/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
- watcher_db_data:/var/lib/postgresql/data - watcher_db_data:/var/lib/postgresql/data
ports: ports:
- "0.0.0.0:15432:5432" - "0.0.0.0:15432:5432"
@ -30,10 +32,12 @@ services:
image: cerc/watcher-mobymask:local image: cerc/watcher-mobymask:local
command: ["sh", "-c", "yarn server"] command: ["sh", "-c", "yarn server"]
volumes: volumes:
- ../common/watcher-ts/mobymask-watcher.toml:/app/packages/mobymask-watcher/environments/local.toml - ../config/watcher-mobymask/mobymask-watcher.toml:/app/watcher-ts/packages/mobymask-watcher/environments/local.toml
ports: ports:
- "0.0.0.0:3001:3001" - "0.0.0.0:3001:3001"
- "0.0.0.0:9001:9001" - "0.0.0.0:9001:9001"
extra_hosts:
- "ipld-eth-server:host-gateway"
healthcheck: healthcheck:
test: ["CMD", "nc", "-v", "localhost", "3001"] test: ["CMD", "nc", "-v", "localhost", "3001"]
interval: 20s interval: 20s
@ -51,9 +55,11 @@ services:
image: cerc/watcher-mobymask:local image: cerc/watcher-mobymask:local
command: ["sh", "-c", "yarn job-runner"] command: ["sh", "-c", "yarn job-runner"]
volumes: volumes:
- ../common/watcher-ts/mobymask-watcher.toml:/app/packages/mobymask-watcher/environments/local.toml - ../config/watcher-mobymask/mobymask-watcher.toml:/app/watcher-ts/packages/mobymask-watcher/environments/local.toml
ports: ports:
- "0.0.0.0:9000:9000" - "0.0.0.0:9000:9000"
extra_hosts:
- "ipld-eth-server:host-gateway"
volumes: volumes:
indexer_db_data: indexer_db_data:

View File

@ -0,0 +1,38 @@
#!/bin/bash
set -e
set -u
function create_user_and_database() {
local database=$1
echo " Creating user and database '$database'"
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
CREATE DATABASE "$database";
GRANT ALL PRIVILEGES ON DATABASE "$database" TO $POSTGRES_USER;
EOSQL
}
function create_extension() {
local database=$(echo $1 | tr ':' ' ' | awk '{print $1}')
local extension=$(echo $1 | tr ':' ' ' | awk '{print $2}')
echo " Creating database '$database' extension '$extension'"
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" "$database" <<-EOSQL
CREATE EXTENSION "$extension";
EOSQL
}
if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then
echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES"
for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do
create_user_and_database $db
done
echo "Multiple databases created"
fi
if [ -n "$POSTGRES_EXTENSION" ]; then
echo "Extension database creation requested: $POSTGRES_EXTENSION"
for db in $(echo $POSTGRES_EXTENSION | tr ',' ' '); do
create_extension $db
done
echo "Extensions created"
fi

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,53 @@
[server]
host = "0.0.0.0"
port = 3001
kind = "active"
# Checkpointing state.
checkpointing = true
# Checkpoint interval in number of blocks.
checkpointInterval = 2000
# IPFS API address (can be taken from the output on running the IPFS daemon).
# ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001"
# Boolean to filter logs by contract.
filterLogs = true
# Max block range for which to return events in eventsInRange GQL query.
# Use -1 for skipping check on block range.
maxEventsBlockRange = -1
[metrics]
host = "0.0.0.0"
port = 9000
[metrics.gql]
port = 9001
[database]
type = "postgres"
host = "watcher-db"
port = 5432
database = "mobymask-watcher"
username = "vdbm"
password = "password"
synchronize = true
logging = false
[upstream]
[upstream.ethServer]
gqlApiEndpoint = "http://ipld-eth-server:8083/graphql"
rpcProviderEndpoint = "http://ipld-eth-server:8082"
blockDelayInMilliSecs = 60000
[upstream.cache]
name = "requests"
enabled = false
deleteOnStart = false
[jobQueue]
dbConnectionString = "postgres://vdbm:password@watcher-db/mobymask-watcher-job-queue"
maxCompletionLagInSecs = 300
jobDelayInMilliSecs = 100
eventsInBatch = 50

View File

@ -10,4 +10,5 @@ prometheus-grafana
laconicd laconicd
fixturenet-laconicd fixturenet-laconicd
fixturenet-eth fixturenet-eth
watcher-mobymask
test test

View File

@ -1,12 +1,14 @@
# Mobymask # Mobymask
## Set up a Mobymask Watcher ## Deploy the Mobymask Watcher
The instructions below show how to deploy a mobymask watcher using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator#install)).
This deployment expects that ipld-eth-server's endpoints are available on the local machine at http://ipld-eth-server.example.com:8083/graphql and http://ipld-eth-server.example.com:8082. More advanced configurations are supported by modifying the watcher's [config file](../../config/watcher-mobymask/mobymask-watcher.toml).
## Clone required repositories ## Clone required repositories
``` ```
$ laconic-so setup-repositories $ laconic-so setup-repositories
``` ```
Checkout required branches: Checkout required branches for the current release:
``` ```
$ cd ~/cerc/assemblyscript $ cd ~/cerc/assemblyscript
$ git checkout ng-integrate-asyncify $ git checkout ng-integrate-asyncify
@ -17,7 +19,18 @@ $ git checkout v0.2.13
``` ```
$ laconic-sh build-containers --include cerc/watcher-mobymask $ laconic-sh build-containers --include cerc/watcher-mobymask
``` ```
This should create a container with tag `cerc/watcher-mobymask` in the local image registry.
## Deploy the stack ## Deploy the stack
First the watcher database has to be initialized. Start only the watcher-db service:
```
$ laconic-so deploy-system --include watcher-mobymask up watcher-db
```
Next find the container's id using `docker ps` then run the following command to initialize the database:
```
$ docker exec -i <watcher-db-container> psql -U vdbm mobymask-watcher < config/watcher-mobymask/mobymask-watcher-db.sql
```
Finally start the remaining containers:
``` ```
$ laconic-sh deploy-system --include watcher-mobymask $ laconic-sh deploy-system --include watcher-mobymask
``` ```
Correct operation should be verified by following the instructions [here](https://github.com/cerc-io/mobymask-watcher/tree/main/mainnet-watcher-only#run), checking GraphQL queries return valid results in the watcher's [playground](http://127.0.0.1:3001/graphql).

View File

@ -1,3 +1,4 @@
version: "1.0"
name: mobymask-watcher name: mobymask-watcher
repos: repos:
- cerc-io/watcher-ts/v0.2.13 - cerc-io/watcher-ts/v0.2.13