Merge pull request #224 from cerc-io/dboreham/update-registry-doc

Update registry deploy instructions

Former-commit-id: 33bbbccef7

app/data/compose/docker-compose-foundry.yml

@@ -1,5 +1,8 @@
 # Add-on pod to include foundry tooling within a fixturenet
 services:
   foundry:
-    image: ghcr.io/foundry-rs/foundry:latest
+    image: cerc/foundry:local
     command: ["while :; do sleep 600; done"]
+    volumes:
+      - ../config/foundry/foundry.toml:/foundry.toml
+      - ./foundry/workspace:/workspace

app/data/compose/docker-compose-watcher-mobymask-v2.yml

@@ -0,0 +1,47 @@
+version: '3.2'
+
+services:
+  mobymask-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=mobymask-watcher,mobymask-watcher-job-queue
+      - POSTGRES_EXTENSION=mobymask-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - mobymask_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "0.0.0.0:15432:5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  mobymask-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      mobymask-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-mobymask-v2:local
+    command: ["sh", "-c", "yarn server"]
+    volumes:
+      - ../config/watcher-mobymask-v2/watcher.toml:/app/packages/mobymask-v2-watcher/environments/local.toml
+      - ../config/watcher-mobymask-v2/.env:/app/packages/peer/.env
+      - ../config/watcher-mobymask-v2/relay-id.json:/app/packages/mobymask-v2-watcher/relay-id.json
+    ports:
+      - "0.0.0.0:3001:3001"
+      - "0.0.0.0:9001:9001"
+      - "0.0.0.0:9090:9090"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "9090"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+
+volumes:
+  mobymask_watcher_db_data:

app/data/config/foundry/foundry.toml

@@ -0,0 +1,2 @@
+[profile.default]
+eth-rpc-url = "http://fixturenet-eth-geth-1:8545"

app/data/config/watcher-mobymask-v2/.env

@@ -0,0 +1 @@
+RELAY="/ip4/127.0.0.1/tcp/9090/http/p2p-webrtc-direct/p2p/12D3KooWSPCsVkHVyLQoCqhu2YRPvvM7o6r6NRYyLM5zeA6Uig5t"

app/data/config/watcher-mobymask-v2/config.json

@@ -0,0 +1,8 @@
+{
+  "relayNodes": [
+    "/ip4/127.0.0.1/tcp/9090/http/p2p-webrtc-direct/p2p/12D3KooWSPCsVkHVyLQoCqhu2YRPvvM7o6r6NRYyLM5zeA6Uig5t"
+  ],
+  "peer": {
+    "enableDebugInfo": true
+  }
+}

app/data/config/watcher-mobymask-v2/relay-id.json

@@ -0,0 +1,5 @@
+{
+  "id": "12D3KooWSPCsVkHVyLQoCqhu2YRPvvM7o6r6NRYyLM5zeA6Uig5t",
+  "privKey": "CAESQGsqG5o4VlWJZM9XlA3MjabyZOXWQ2MLZU5AhBQsjXGt9iSlGtTuNOrHX5xSRgLBxLuMoqWsjGxE/dDB9c46RI8=",
+  "pubKey": "CAESIPYkpRrU7jTqx1+cUkYCwcS7jKKlrIxsRP3QwfXOOkSP"
+}

app/data/config/watcher-mobymask-v2/watcher.toml

@@ -0,0 +1,70 @@
+[server]
+  host = "0.0.0.0"
+  port = 3001
+  kind = "lazy"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # Enable state creation
+  enableState = true
+
+  # Boolean to filter logs by contract.
+  filterLogs = true
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = -1
+
+  [server.p2p]
+    enableRelay = true
+    enablePeer = false
+
+    [server.p2p.relay]
+      host = "0.0.0.0"
+      port = 9090
+      relayPeers = []
+      peerIdFile = './relay-id.json'
+      enableDebugInfo = true
+
+    [server.p2p.peer]
+      relayMultiaddr = '/ip4/mobymask-watcher-server/tcp/9090/http/p2p-webrtc-direct/p2p/12D3KooWSPCsVkHVyLQoCqhu2YRPvvM7o6r6NRYyLM5zeA6Uig5t'
+      pubSubTopic = 'mobymask'
+      peerIdFile = './peer-id.json'
+      enableDebugInfo = true
+
+[metrics]
+  host = "0.0.0.0"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "mobymask-watcher-db"
+  port = 5432
+  database = "mobymask-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+
+[upstream]
+  [upstream.ethServer]
+    gqlApiEndpoint = "http://ipld-eth-server:8083/graphql"
+    rpcProviderEndpoint = "http://ipld-eth-server:8082"
+    blockDelayInMilliSecs = 60000
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@mobymask-watcher-db/mobymask-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50

app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh

@@ -24,7 +24,7 @@ if [ -z "$GETH_BASE_URL" ]; then
 fi
 
 function inc_status() {
-  echo " DONE!"
+  echo " done"
   STATUS=$((STATUS + 1))
   if [ $STATUS -lt ${#STATUSES[@]} ]; then
     echo -n "Waiting for ${STATUSES[$STATUS]}..."

app/data/container-build/cerc-foundry/build.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+# Build a local version of the foundry-rs/foundry image
+docker build -t cerc/foundry:local ${CERC_REPO_BASE_DIR}/foundry

app/data/container-build/cerc-mobymask-ui/Dockerfile

@@ -0,0 +1,12 @@
+FROM node:18.15.0-alpine3.16
+
+RUN apk --update --no-cache add make git
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Building mobymask-ui" && \
+    npm install
+
+WORKDIR /app

app/data/container-build/cerc-mobymask-ui/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Build cerc/mobymask-ui
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/mobymask-ui:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/mobymask-ui

app/data/container-build/cerc-react-peer/Dockerfile

@@ -0,0 +1,12 @@
+FROM node:18.15.0-alpine3.16
+
+RUN apk --update --no-cache add make git
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Building react-peer" && \
+    yarn && yarn workspace @cerc-io/react-peer build
+
+WORKDIR /app

app/data/container-build/cerc-react-peer/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Build cerc/react-peer
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/react-peer:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/react-peer

app/data/container-build/cerc-watcher-mobymask-v2/Dockerfile

@@ -0,0 +1,20 @@
+FROM ubuntu:22.04
+
+RUN apt-get update \
+  && apt-get install -y curl gnupg build-essential \
+  && curl --silent --location https://deb.nodesource.com/setup_18.x | bash - \
+  && apt-get update \
+  && apt-get install -y nodejs git \
+  && node -v
+
+RUN corepack enable \
+  && yarn --version
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Building watcher-ts" && \
+    yarn && yarn build
+
+WORKDIR /app/packages/mobymask-v2-watcher

app/data/container-build/cerc-watcher-mobymask-v2/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Build cerc/watcher-mobymask-v2
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/watcher-mobymask-v2:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/watcher-ts

app/data/container-build/foundry-rs-foundry/build.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-# Build foundry-rs/foundry
-# HACK below : TARGETARCH needs to be derived from the host environment
-docker build -t foundry-rs/foundry:local ${CERC_REPO_BASE_DIR}/foundry

app/data/container-image-list.txt

@@ -1,4 +1,4 @@
-foundry-rs/foundry
+cerc/foundry
 cerc/test-contract
 cerc/eth-statediff-fill-service
 cerc/eth-statediff-service
@@ -18,6 +18,9 @@ cerc/watcher-erc20
 cerc/watcher-erc721
 cerc/watcher-uniswap-v3
 cerc/uniswap-v3-info
+cerc/watcher-mobymask-v2
+cerc/react-peer
+cerc/mobymask-ui
 cerc/test-container
 cerc/eth-probe
 cerc/builder-js

app/data/pod-list.txt

@@ -14,6 +14,7 @@ watcher-mobymask
 watcher-erc20
 watcher-erc721
 watcher-uniswap-v3
+watcher-mobymask-v2
 test
 eth-probe
 keycloak

app/data/repository-list.txt

@@ -11,9 +11,11 @@ cerc-io/laconic-sdk
 cerc-io/laconic-registry-cli
 cerc-io/mobymask-watcher
 cerc-io/watcher-ts
+cerc-io/react-peer
+cerc-io/mobymask-ui
 vulcanize/uniswap-watcher-ts
 vulcanize/uniswap-v3-info
 vulcanize/assemblyscript
 cerc-io/eth-probe
 cerc-io/tx-spammer
-foundry-rs/foundry
+dboreham/foundry

app/data/stacks/build-support/README.md

@@ -24,8 +24,31 @@ $ laconic-so --stack build-support build-containers --exclude cerc/builder-gerbi
 ```
 $ laconic-so --stack package-registry setup-repositories
 $ laconic-so --stack package-registry deploy up
+[+] Running 3/3
+ ⠿ Network laconic-aecc4a21d3a502b14522db97d427e850_gitea       Created                                                                                    0.0s
+ ⠿ Container laconic-aecc4a21d3a502b14522db97d427e850-db-1      Started                                                                                    1.2s
+ ⠿ Container laconic-aecc4a21d3a502b14522db97d427e850-server-1  Started                                                                                    1.9s
+New user 'gitea_admin' has been successfully created!
 This is your gitea access token: 84fe66a73698bf11edbdccd0a338236b7d1d5c45. Keep it safe and secure, it can not be fetched again from gitea.
+To use with laconic-so set this environment variable: export CERC_NPM_AUTH_TOKEN=3e493e77b3e83fe9e882f7e3a79dd4d5441c308b
+Created the organization cerc-io
+Gitea was configured to use host name: gitea.local, ensure that this resolves to localhost, e.g. with sudo vi /etc/hosts
+Success, gitea is properly initialized
+$
 ```
+### 3. Configure the hostname gitea.local
+How to do this is OS-dependent but usually involves editing a `hosts` file. For example on Linux add this line to the file `/etc/hosts` (needs sudo):
+```
+127.0.0.1       gitea.local
+```
+Test with:
+```
+$ ping gitea.local
+PING gitea.local (127.0.0.1) 56(84) bytes of data.
+64 bytes from localhost (127.0.0.1): icmp_seq=1 ttl=64 time=0.147 ms
+64 bytes from localhost (127.0.0.1): icmp_seq=2 ttl=64 time=0.033 ms
+```
+
 Now npm packages can be built:
 ### Build npm Packages
 Ensure that `CERC_NPM_AUTH_TOKEN` is set with the token printed above when the package-registry stack was deployed (the actual token value will be different than shown in this example):

app/data/stacks/fixturenet-eth/README.md

@@ -20,27 +20,31 @@ This should create several container images in the local image registry:
 
 ## Deploy the stack
 ```
-$ laconic-so --stack fixturenet-eth deploy-system up
+$ laconic-so --stack fixturenet-eth deploy up
 ```
 
 ## Check status
 
 ```
-$ container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh
+$ laconic-so --stack fixturenet-eth deploy exec fixturenet-eth-bootnode-lighthouse /scripts/status-internal.sh
-Waiting for geth to generate DAG ..................................................... DONE!
+Waiting for geth to generate DAG.... done
-Waiting for beacon phase0 .... DONE!
+Waiting for beacon phase0.... done
-Waiting for beacon altair .... DONE!
+Waiting for beacon altair.... done
-Waiting for beacon bellatrix pre-merge .... DONE!
+Waiting for beacon bellatrix pre-merge.... done
-Waiting for beacon bellatrix merge .... DONE!
+Waiting for beacon bellatrix merge.... done
 
-$ docker ps -f 'name=laconic' --format 'table {{.Names}}\t{{.Ports}}'  | cut -d'-' -f3- | sort
+$ laconic-so --stack fixturenet-eth deploy ps
-NAMES                                                                           PORTS
+Running containers:
-fixturenet-eth-bootnode-geth-1         8545-8546/tcp, 30303/udp, 0.0.0.0:55847->9898/tcp, 0.0.0.0:55848->30303/tcp
+id: c6538b60c0328dadfa2c5585c4d09674a6a13e6d712ff1cd82a26849e4e5679b, name: laconic-b12fa16e999821562937781f8ab0b1e8-fixturenet-eth-bootnode-geth-1, ports: 0.0.0.0:58909->30303/tcp, 0.0.0.0:58910->9898/tcp
-fixturenet-eth-bootnode-lighthouse-1
+id: 5b70597a8211bc7e78d33e50486cb565a7f4a9ce581ce150b3bb450e342bdeda, name: laconic-b12fa16e999821562937781f8ab0b1e8-fixturenet-eth-bootnode-lighthouse-1, ports:
-fixturenet-eth-geth-1-1                8546/tcp, 30303/tcp, 30303/udp, 0.0.0.0:55851->8545/tcp
+id: 19ed78867b6c534d893835cdeb1e89a9ea553b8e8c02ab02468e4bd1563a340f, name: laconic-b12fa16e999821562937781f8ab0b1e8-fixturenet-eth-geth-1-1, ports: 0.0.0.0:58911->40000/tcp, 0.0.0.0:58912->6060/tcp, 0.0.0.0:58913->8545/tcp
-fixturenet-eth-geth-2-1                8545-8546/tcp, 30303/tcp, 30303/udp
+id: 8da0e30a1ce33122d8fd2225e4d26c7f30eb4bfbfa743f2af04d9db5d0bf7fa6, name: laconic-b12fa16e999821562937781f8ab0b1e8-fixturenet-eth-geth-2-1, ports:
-fixturenet-eth-lighthouse-1-1          0.0.0.0:55858->8001/tcp
+id: 387a42a14971034588ba9aeb9b9e2ca7fc0cc61b96f8fe8c2ab770c9d6fb1e0f, name: laconic-b12fa16e999821562937781f8ab0b1e8-fixturenet-eth-lighthouse-1-1, ports: 0.0.0.0:58917->8001/tcp
-fixturenet-eth-lighthouse-2-1
+id: de5115bf89087bae03b291664a73ffe3554fe23e79e4b8345e088b040d5580ac, name: laconic-b12fa16e999821562937781f8ab0b1e8-fixturenet-eth-lighthouse-2-1, ports:
+id: 2a7e5a0fb2be7fc9261a7b725a40818facbbe6d0cb2497d82c0e02de0a8e959b, name: laconic-b12fa16e999821562937781f8ab0b1e8-foundry-1, ports:
+
+$ laconic-so --stack fixturenet-eth deploy exec foundry "cast block-number"
+3
 ```
 
 ## Additional pieces
@@ -69,11 +73,11 @@ $ laconic-so deploy-system --include db,fixturenet-eth,ipld-eth-server,ipld-eth-
 # Status
 
 $ container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh
-Waiting for geth to generate DAG.... DONE!
+Waiting for geth to generate DAG.... done
-Waiting for beacon phase0.... DONE!
+Waiting for beacon phase0.... done
-Waiting for beacon altair.... DONE!
+Waiting for beacon altair.... done
-Waiting for beacon bellatrix pre-merge.... DONE!
+Waiting for beacon bellatrix pre-merge.... done
-Waiting for beacon bellatrix merge.... DONE!
+Waiting for beacon bellatrix merge.... done
 
 $ docker ps -f 'name=laconic' --format 'table {{.Names}}\t{{.Ports}}'  | cut -d'-' -f3- | sort
 NAMES                                                                           PORTS

app/data/stacks/fixturenet-eth/stack.yml

@@ -4,11 +4,13 @@ decription: "Ethereum Fixturenet"
 repos:
   - cerc-io/go-ethereum
   - cerc-io/tx-spammer
+  - dboreham/foundry
 containers:
   - cerc/go-ethereum
   - cerc/lighthouse
   - cerc/fixturenet-eth-geth
   - cerc/fixturenet-eth-lighthouse
+  - cerc/foundry
 pods:
   - fixturenet-eth
   - foundry

app/data/stacks/fixturenet-laconicd/README.md

@@ -44,7 +44,5 @@ $ laconic-so --stack fixturenet-laconicd deploy logs
 ```
 ## 6. Test with the Registry CLI
 ```
-$ laconic-so --stack fixturenet-laconicd deploy exec laconicd "sh /docker-entrypoint-scripts.d/export-mykey.sh"
-$ laconic-so --stack fixturenet-laconicd deploy exec cli "sh /scripts/import-key.sh xxxxxxxxxxxxxxxxxxxxxxx"
 $ laconic-so --stack fixturenet-laconicd deploy exec cli "laconic cns status"
 ```

app/data/stacks/mobymask-v2/README.md

@@ -0,0 +1,61 @@
+# MobyMask v2 watcher
+
+Instructions to deploy MobyMask v2 watcher stack using [laconic-stack-orchestrator](/README.md#install)
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack mobymask-v2 setup-repositories
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack mobymask-v2 build-containers
+```
+
+This should create the required docker images in the local image registry.
+
+Deploy the stack:
+
+```bash
+laconic-so --stack mobymask-v2 deploy-system up
+```
+
+## Tests
+
+Find the watcher container's id:
+
+```bash
+docker ps | grep "cerc/watcher-mobymask-v2:local"
+```
+
+Example output
+
+```
+8b38e9a64d7e   cerc/watcher-mobymask-v2:local   "sh -c 'yarn server'"    35 seconds ago   Up 14 seconds (health: starting)   0.0.0.0:3001->3001/tcp, 0.0.0.0:9001->9001/tcp, 0.0.0.0:9090->9090/tcp   laconic-aeb84676de2b0a7671ae90d537fc7d26-mobymask-watcher-server-1
+```
+
+In above output the container ID is `8b38e9a64d7e`
+
+Export it for later use:
+
+```bash
+export CONTAINER_ID=<CONTAINER_ID>
+```
+
+Run the peer tests:
+
+```bash
+docker exec -w /app/packages/peer $CONTAINER_ID yarn test
+```
+
+## Clean up
+
+To stop all the services running in background run:
+
+```bash
+laconic-so --stack mobymask-v2 deploy-system down
+```

app/data/stacks/mobymask-v2/stack.yml

@@ -0,0 +1,11 @@
+version: "1.0"
+name: mobymask-v2
+repos:
+  - cerc-io/watcher-ts
+  - cerc-io/react-peer
+  - cerc-io/mobymask-ui
+containers:
+  - cerc/watcher-mobymask-v2
+  - cerc/mobymask-ui
+pods:
+  - watcher-mobymask-v2

app/data/version.txt

@@ -1,2 +1,2 @@
 # This file should be re-generated running: scripts/update-version-file.sh script
-v1.0.25-75376d7
+v1.0.27-7831078

app/deploy_system.py

@@ -22,7 +22,7 @@ import sys
 from dataclasses import dataclass
 from decouple import config
 import subprocess
-from python_on_whales import DockerClient
+from python_on_whales import DockerClient, DockerException
 import click
 import importlib.resources
 from pathlib import Path
@@ -83,7 +83,10 @@ def command(ctx, include, exclude, cluster, command, extra_args):
             container_exec_env = _make_runtime_env(ctx.obj)
             if verbose:
                 print(f"Running compose exec {service_name} {command_to_exec}")
-            docker.compose.execute(service_name, command_to_exec, envs=container_exec_env)
+            try:
+                docker.compose.execute(service_name, command_to_exec, envs=container_exec_env)
+            except DockerException as error:
+                print(f"container command returned error exit status")
         elif command == "port":
             if extra_args_list is None or len(extra_args_list) < 2:
                 print("Usage: port <service> <exposed-port>")
@@ -301,7 +304,7 @@ def _orchestrate_cluster_config(ctx, cluster_config, docker, container_exec_env)
                     print(f"fetched source value: {source_value}")
                 destination_output = docker.compose.execute(pd.destination_container,
                                                             ["sh", "-c",
-                                                             f"sh /scripts/import-{pd.destination_variable}.sh {pd.source_variable}"],
+                                                             f"sh /scripts/import-{pd.destination_variable}.sh {source_value}"],
                                                             tty=False,
                                                             envs=container_exec_env)
                 if ctx.debug: