Merge pull request #234 from cerc-io/dboreham/lirewine-builds

Add support for lirewine npm package build and consumption

Former-commit-id: ebe1dfa4bf

README.md

@@ -8,11 +8,10 @@ Stack Orchestrator allows building and deployment of a Laconic Stack on a single
 
 Ensure that the following are already installed:
 
-- [Python3](https://wiki.python.org/moin/BeginnersGuide/Download): `python3 --version` >= `3.10.8`
+- [Python3](https://wiki.python.org/moin/BeginnersGuide/Download): `python3 --version` >= `3.8.10` (the Python3 shipped in Ubuntu 20+ is good to go)
 - [Docker](https://docs.docker.com/get-docker/): `docker --version` >= `20.10.21`
-- [Docker Compose](https://docs.docker.com/compose/install/): `docker-compose --version` >= `2.13.0`
 
-Note: if installing docker-compose via package manager (as opposed to Docker Desktop), you must [install the plugin](https://docs.docker.com/compose/install/linux/#install-the-plugin-manually), e.g., on Linux:
+Note: if installing docker-compose via package manager on Linux (as opposed to Docker Desktop), you must [install the plugin](https://docs.docker.com/compose/install/linux/#install-the-plugin-manually), e.g. :
 
 ```bash
 mkdir -p ~/.docker/cli-plugins
@@ -20,39 +19,27 @@ curl -SL https://github.com/docker/compose/releases/download/v2.11.2/docker-comp
 chmod +x ~/.docker/cli-plugins/docker-compose
 ```
 
-Next, download the latest release from [this page](https://github.com/cerc-io/stack-orchestrator/tags), into a suitable directory (e.g. `~/bin`):
+Next decide on a directory where you would like to put the stack-orchestrator program. Typically this would be 
+a "user" binary directory such as `~/bin` or perhaps `/usr/local/laconic` or possibly just the current working directory.
+
+Now, having selected that directory, download the latest release from [this page](https://github.com/cerc-io/stack-orchestrator/tags) into it (we're using `~/bin` below for concreteness but edit to suit if you selected a different directory). Also be sure that the destination directory exists and is writable:
 
 ```bash
 curl -L -o ~/bin/laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
 ```
 
-Give it permissions:
+Give it execute permissions:
 ```bash
 chmod +x ~/bin/laconic-so
 ```
 
 Ensure `laconic-so` is on the [`PATH`](https://unix.stackexchange.com/a/26059)
 
-Verify operation:
+Verify operation (your version will probably be different, just check here that you see some version outut and not an error):
 
 ```
-laconic-so --help
-Usage: python -m laconic-so [OPTIONS] COMMAND [ARGS]...
-
-  Laconic Stack Orchestrator
-
-Options:
-  --quiet
-  --verbose
-  --dry-run
-  --local-stack
-  -h, --help     Show this message and exit.
-
-Commands:
-  build-containers    build the set of containers required for a complete...
-  build-npms          build the set of npm packages required for a...  
-  deploy-system       deploy a stack
-  setup-repositories  git clone the set of repositories required to build...
+laconic-so version
+Version: v1.0.27-7831078
 ```
 
 ## Usage
@@ -64,25 +51,25 @@ Three sub-commands: `setup-repositories`, `build-containers` and `deploy-system`
 Clone the set of git repositories necessary to build a system:
 
 ```bash
-laconic-so --verbose setup-repositories --include cerc-io/go-ethereum,cerc-io/ipld-eth-db,cerc-io/ipld-eth-server,cerc-io/watcher-ts
+laconic-so --stack erc20 setup-repositories
 ```
 
-This will default to `~/cerc` or - if set - the environment variable `CERC_REPO_BASE_DIR`
+This will default to cloning git reposiories into: `~/cerc` or - if set - the environment variable `CERC_REPO_BASE_DIR`
 
 ### Build Containers
 
 Build the set of docker container images required to run a system. It takes around 10 minutes to build all the containers from scratch.
 
 ```bash
-laconic-so --verbose build-containers --include cerc/go-ethereum,cerc/go-ethereum-foundry,cerc/ipld-eth-db,cerc/ipld-eth-server,cerc/watcher-erc20
+laconic-so --stack erc20 build-containers
 ```
 
 ### Deploy System
 
-Uses `docker-compose` to deploy a system (with most recently built container images).
+Uses `docker compose` to deploy a system (with most recently built container images).
 
 ```bash
-laconic-so --verbose deploy-system --include ipld-eth-db,go-ethereum-foundry,ipld-eth-server,watcher-erc20 up
+laconic-so --stack erc20 deploy-system up
 ```
 
 Check out he GraphQL playground here: [http://localhost:3002/graphql](http://localhost:3002/graphql)
@@ -92,7 +79,7 @@ See the [erc20 watcher demo](/app/data/stacks/erc20) to continue further.
 ### Cleanup
 
 ```bash
-laconic-so --verbose deploy-system --include ipld-eth-db,go-ethereum-foundry,ipld-eth-server,watcher-erc20 down
+laconic-so --stack erc20 deploy-system down
 ```
 
 ## Contributing

app/build_containers.py

@@ -83,7 +83,10 @@ def command(ctx, include, exclude):
     container_build_env = {
         "CERC_NPM_URL": "http://gitea.local:3000/api/packages/cerc-io/npm/",
         "CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default="<token-not-supplied>"),
-        "CERC_REPO_BASE_DIR": dev_root_path
+        "CERC_REPO_BASE_DIR": dev_root_path,
+        "CERC_HOST_UID": f"{os.getuid()}",
+        "CERC_HOST_GID": f"{os.getgid()}",
+        "DOCKER_BUILDKIT": "0"
     }
 
     def process_container(container):
@@ -106,7 +109,7 @@ def command(ctx, include, exclude):
             build_command = os.path.join(container_build_dir, "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}"
         if not dry_run:
             if verbose:
-                print(f"Executing: {build_command}")
+                print(f"Executing: {build_command} with environment: {container_build_env}")
             build_result = subprocess.run(build_command, shell=True, env=container_build_env)
             if verbose:
                 print(f"Return code is: {build_result.returncode}")

app/build_npms.py

@@ -116,7 +116,11 @@ def command(ctx, include, exclude):
         if not dry_run:
             if verbose:
                 print(f"Executing: {build_command}")
-            envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token} | ({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
+            # Originally we used the PEP 584 merge operator:
+            # envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token} | ({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
+            # but that isn't available in Python 3.8 (default in Ubuntu 20) so for now we use dict.update:
+            envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token}
+            envs.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
             try:
                 docker.run(builder_js_image_name,
                            remove=True,

app/data/compose/docker-compose-fixturenet-laconicd.yml

@@ -5,8 +5,9 @@ services:
     image: cerc/laconicd:local
     command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
     volumes:
-      # TODO: look at folding this script into the container
+      # TODO: look at folding these scripts into the container
       - ../config/fixturenet-laconicd/create-fixturenet.sh:/docker-entrypoint-scripts.d/create-fixturenet.sh
+      - ../config/fixturenet-laconicd/export-mykey.sh:/docker-entrypoint-scripts.d/export-mykey.sh
     # TODO: determine which of the ports below is really needed
     ports:
       - "6060"
@@ -18,4 +19,7 @@ services:
       - "9090"
       - "9091"
       - "1317"
-
+  cli:
+    image: cerc/laconic-registry-cli:local
+    volumes:
+      - ../config/fixturenet-laconicd/registry-cli-config-template.yml:/registry-cli-config-template.yml

app/data/compose/docker-compose-foundry.yml

@@ -0,0 +1,8 @@
+# Add-on pod to include foundry tooling within a fixturenet
+services:
+  foundry:
+    image: cerc/foundry:local
+    command: ["while :; do sleep 600; done"]
+    volumes:
+      - ../config/foundry/foundry.toml:/foundry.toml
+      - ./foundry/workspace:/workspace

app/data/compose/docker-compose-watcher-mobymask-v2.yml

@@ -0,0 +1,47 @@
+version: '3.2'
+
+services:
+  mobymask-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=mobymask-watcher,mobymask-watcher-job-queue
+      - POSTGRES_EXTENSION=mobymask-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - mobymask_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "0.0.0.0:15432:5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  mobymask-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      mobymask-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-mobymask-v2:local
+    command: ["sh", "-c", "yarn server"]
+    volumes:
+      - ../config/watcher-mobymask-v2/watcher.toml:/app/packages/mobymask-v2-watcher/environments/local.toml
+      - ../config/watcher-mobymask-v2/.env:/app/packages/peer/.env
+      - ../config/watcher-mobymask-v2/relay-id.json:/app/packages/mobymask-v2-watcher/relay-id.json
+    ports:
+      - "0.0.0.0:3001:3001"
+      - "0.0.0.0:9001:9001"
+      - "0.0.0.0:9090:9090"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "9090"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+
+volumes:
+  mobymask_watcher_db_data:

app/data/config/fixturenet-laconicd/create-fixturenet.sh

@@ -1,8 +1,8 @@
-#!/bin/sh
-
-# Originally from: https://github.com/cerc-io/laconicd/blob/main/init.sh
-# TODO: fold this back into the laconicd repo
+#!/bin/bash
 
+# TODO: this file is now an unmodified copy of cerc-io/laconicd/init.sh
+# so we should have a mechanism to bundle it inside the container rather than link from here
+# at deploy time.
 
 KEY="mykey"
 CHAINID="laconic_9000-1"
@@ -10,7 +10,7 @@ MONIKER="localtestnet"
 KEYRING="test"
 KEYALGO="eth_secp256k1"
 LOGLEVEL="info"
-# to trace evm
+# trace evm
 TRACE="--trace"
 # TRACE=""
 
@@ -28,7 +28,7 @@ laconicd config chain-id $CHAINID
 # if $KEY exists it should be deleted
 laconicd keys add $KEY --keyring-backend $KEYRING --algo $KEYALGO
 
-# Set moniker and chain-id for laconic (Moniker can be anything, chain-id must be an integer)
+# Set moniker and chain-id for Ethermint (Moniker can be anything, chain-id must be an integer)
 laconicd init $MONIKER --chain-id $CHAINID
 
 # Change parameter token denominations to aphoton
@@ -37,28 +37,28 @@ cat $HOME/.laconicd/config/genesis.json | jq '.app_state["crisis"]["constant_fee
 cat $HOME/.laconicd/config/genesis.json | jq '.app_state["gov"]["deposit_params"]["min_deposit"][0]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
 cat $HOME/.laconicd/config/genesis.json | jq '.app_state["mint"]["params"]["mint_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
 # Custom modules
-cat $HOME/.laconicd/config/genesis.json | jq '.app_state["nameservice"]["params"]["record_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-cat $HOME/.laconicd/config/genesis.json | jq '.app_state["nameservice"]["params"]["authority_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-cat $HOME/.laconicd/config/genesis.json | jq '.app_state["nameservice"]["params"]["authority_auction_commit_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-cat $HOME/.laconicd/config/genesis.json | jq '.app_state["nameservice"]["params"]["authority_auction_reveal_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-cat $HOME/.laconicd/config/genesis.json | jq '.app_state["nameservice"]["params"]["authority_auction_minimum_bid"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commit_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveal_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_minimum_bid"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
 
-if [[ "$TEST_NAMESERVICE_EXPIRY" == "true" ]]; then
+if [[ "$TEST_REGISTRY_EXPIRY" == "true" ]]; then
   echo "Setting timers for expiry tests."
 
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["nameservice"]["params"]["record_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["nameservice"]["params"]["authority_grace_period"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["nameservice"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
 fi
 
 if [[ "$TEST_AUCTION_ENABLED" == "true" ]]; then
   echo "Enabling auction and setting timers."
 
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["nameservice"]["params"]["authority_auction_enabled"]=true' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["nameservice"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["nameservice"]["params"]["authority_grace_period"]="300s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["nameservice"]["params"]["authority_auction_commits_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["nameservice"]["params"]["authority_auction_reveals_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_enabled"]=true' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="300s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commits_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveals_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
 fi
 
 # increase block time (?)

app/data/config/fixturenet-laconicd/export-mykey.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo y | laconicd keys export mykey --unarmored-hex --unsafe

app/data/config/fixturenet-laconicd/registry-cli-config-template.yml

@@ -0,0 +1,7 @@
+services:
+  cns:
+    restEndpoint: 'http://laconicd:1317'
+    gqlEndpoint: 'http://laconicd:9473/api'
+    userKey: REPLACE_WITH_MYKEY
+    bondId:
+    chainId: laconic_9000-1

app/data/config/foundry/foundry.toml

@@ -0,0 +1,2 @@
+[profile.default]
+eth-rpc-url = "http://fixturenet-eth-geth-1:8545"

app/data/config/watcher-mobymask-v2/.env

@@ -0,0 +1 @@
+RELAY="/ip4/127.0.0.1/tcp/9090/http/p2p-webrtc-direct/p2p/12D3KooWSPCsVkHVyLQoCqhu2YRPvvM7o6r6NRYyLM5zeA6Uig5t"

app/data/config/watcher-mobymask-v2/config.json

@@ -0,0 +1,8 @@
+{
+  "relayNodes": [
+    "/ip4/127.0.0.1/tcp/9090/http/p2p-webrtc-direct/p2p/12D3KooWSPCsVkHVyLQoCqhu2YRPvvM7o6r6NRYyLM5zeA6Uig5t"
+  ],
+  "peer": {
+    "enableDebugInfo": true
+  }
+}

app/data/config/watcher-mobymask-v2/relay-id.json

@@ -0,0 +1,5 @@
+{
+  "id": "12D3KooWSPCsVkHVyLQoCqhu2YRPvvM7o6r6NRYyLM5zeA6Uig5t",
+  "privKey": "CAESQGsqG5o4VlWJZM9XlA3MjabyZOXWQ2MLZU5AhBQsjXGt9iSlGtTuNOrHX5xSRgLBxLuMoqWsjGxE/dDB9c46RI8=",
+  "pubKey": "CAESIPYkpRrU7jTqx1+cUkYCwcS7jKKlrIxsRP3QwfXOOkSP"
+}

app/data/config/watcher-mobymask-v2/watcher.toml

@@ -0,0 +1,70 @@
+[server]
+  host = "0.0.0.0"
+  port = 3001
+  kind = "lazy"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # Enable state creation
+  enableState = true
+
+  # Boolean to filter logs by contract.
+  filterLogs = true
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = -1
+
+  [server.p2p]
+    enableRelay = true
+    enablePeer = false
+
+    [server.p2p.relay]
+      host = "0.0.0.0"
+      port = 9090
+      relayPeers = []
+      peerIdFile = './relay-id.json'
+      enableDebugInfo = true
+
+    [server.p2p.peer]
+      relayMultiaddr = '/ip4/mobymask-watcher-server/tcp/9090/http/p2p-webrtc-direct/p2p/12D3KooWSPCsVkHVyLQoCqhu2YRPvvM7o6r6NRYyLM5zeA6Uig5t'
+      pubSubTopic = 'mobymask'
+      peerIdFile = './peer-id.json'
+      enableDebugInfo = true
+
+[metrics]
+  host = "0.0.0.0"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "mobymask-watcher-db"
+  port = 5432
+  database = "mobymask-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+
+[upstream]
+  [upstream.ethServer]
+    gqlApiEndpoint = "http://ipld-eth-server:8083/graphql"
+    rpcProviderEndpoint = "http://ipld-eth-server:8082"
+    blockDelayInMilliSecs = 60000
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@mobymask-watcher-db/mobymask-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50

app/data/container-build/cerc-builder-js/Dockerfile

@@ -1,14 +1,30 @@
 # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
+# Which depends on: https://github.com/nodejs/docker-node/blob/main/Dockerfile-debian.template
 # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
 ARG VARIANT=16-bullseye
 FROM node:${VARIANT}
 
+# Set these args to change the uid/gid for the base container's "node" user to match that of the host user (so bind mounts work as expected).
+ARG CERC_HOST_UID=1000
+ARG CERC_HOST_GID=1000
+# Make these values available at runtime to allow a consistency check.
+ENV HOST_UID=${CERC_HOST_UID}
+ENV HOST_GID=${CERC_HOST_GID}
+
 ARG USERNAME=node
 ARG NPM_GLOBAL=/usr/local/share/npm-global
 
 # Add NPM global to PATH.
 ENV PATH=${NPM_GLOBAL}/bin:${PATH}
 
+RUN \
+    if [ ${CERC_HOST_GID} -ne 1000 ] ; then \
+        groupmod -g ${CERC_HOST_GID} ${USERNAME} ; \
+    fi \
+    && if [ ${CERC_HOST_UID} -ne 1000 ] ; then \
+        usermod -u ${CERC_HOST_UID} -g ${CERC_HOST_GID} ${USERNAME} && chown ${CERC_HOST_UID}:${CERC_HOST_GID} /home/${USERNAME} ; \
+    fi
+
 RUN \
     # Configure global npm install location, use group to adapt to UID/GID changes
     if ! cat /etc/group | grep -e "^npm:" > /dev/null 2>&1; then groupadd -r npm; fi \
@@ -39,6 +55,7 @@ RUN mkdir /scripts
 COPY build-npm-package.sh /scripts
 COPY yarn-local-registry-fixup.sh /scripts
 COPY build-npm-package-local-dependencies.sh /scripts
+COPY check-uid.sh /scripts
 ENV PATH="${PATH}:/scripts"
 
 COPY entrypoint.sh .

app/data/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # Usage: build-npm-package-local-dependencies.sh <registry-url> <publish-with-this-version>
 # Runs build-npm-package.sh after first fixing up yarn.lock to use a local
-# npm registry for all packages in a spcific scope (currently @cerc-io)
+# npm registry for all packages in a spcific scope (currently @cerc-io and @lirewine)
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
@@ -17,18 +17,21 @@ fi
 set -e
 local_npm_registry_url=$1
 package_publish_version=$2
-# TODO: make this a paramater and allow a list of scopes
-npm_scope_for_local="@cerc-io"
-# We need to configure the local registry 
-npm config set ${npm_scope_for_local}:registry ${local_npm_registry_url}
-npm config set -- ${local_npm_registry_url}:_authToken ${CERC_NPM_AUTH_TOKEN}
-# Find the set of dependencies from the specified scope
-mapfile -t dependencies_from_scope < <(cat package.json | jq -r '.dependencies | with_entries(if (.key|test("^'${npm_scope_for_local}'/.*$")) then ( {key: .key, value: .value } ) else empty end ) | keys[]')
-echo "Fixing up dependencies"
-for package in "${dependencies_from_scope[@]}"
+# If we need to handle an additional scope, add it to the list below:
+npm_scopes_to_handle=("@cerc-io" "@lirewine")
+for npm_scope_for_local in ${npm_scopes_to_handle[@]}
 do
-    echo "Fixing up package ${package}"
-    yarn-local-registry-fixup.sh $package ${local_npm_registry_url}
+    # We need to configure the local registry
+    npm config set ${npm_scope_for_local}:registry ${local_npm_registry_url}
+    npm config set -- ${local_npm_registry_url}:_authToken ${CERC_NPM_AUTH_TOKEN}
+    # Find the set of dependencies from the specified scope
+    mapfile -t dependencies_from_scope < <(cat package.json | jq -r '.dependencies | with_entries(if (.key|test("^'${npm_scope_for_local}'/.*$")) then ( {key: .key, value: .value } ) else empty end ) | keys[]')
+    echo "Fixing up dependencies in scope ${npm_scope_for_local}"
+    for package in "${dependencies_from_scope[@]}"
+    do
+        echo "Fixing up package ${package}"
+        yarn-local-registry-fixup.sh $package ${local_npm_registry_url}
+    done
 done
 echo "Running build"
 build-npm-package.sh ${local_npm_registry_url} ${package_publish_version}

app/data/container-build/cerc-builder-js/build-npm-package.sh

@@ -22,8 +22,8 @@ set -e
 # Get the name of this package from package.json since we weren't passed that
 package_name=$( cat package.json | jq -r .name )
 local_npm_registry_url=$1
-npm config set @lirewine:registry ${local_npm_registry_url}
 npm config set @cerc-io:registry ${local_npm_registry_url}
+npm config set @lirewine:registry ${local_npm_registry_url}
 npm config set -- ${local_npm_registry_url}:_authToken ${CERC_NPM_AUTH_TOKEN}
 # First check if the version of this package we're trying to build already exists in the registry
 package_exists=$( yarn info --json ${package_name}@${package_publish_version} 2>/dev/null | jq -r .data.dist.tarball )

app/data/container-build/cerc-builder-js/check-uid.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Make the container usable for uid/gid != 1000
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+current_uid=$(id -u)
+current_gid=$(id -g)
+# Don't check if running as root
+if [[ ${current_uid} == 0 ]]; then
+    exit 0
+fi
+# Check the current uid/gid vs the uid/gid used to build the container.
+# We do this because both bind mounts and npm tooling require the uid/gid to match.
+if [[ ${current_gid} != ${HOST_GID} ]]; then
+    echo "Warning: running with gid: ${current_gid} which is not the gid for which this container was built (${HOST_GID})"
+    exit 0
+fi
+if [[ ${current_uid} != ${HOST_UID} ]]; then
+    echo "Warning: running with gid: ${current_uid} which is not the uid for which this container was built (${HOST_UID})"
+    exit 0
+fi

app/data/container-build/cerc-builder-js/entrypoint.sh

@@ -1,2 +1,3 @@
 #!/bin/sh
+/scripts/check-uid.sh
 exec "$@"

app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh

@@ -24,7 +24,7 @@ if [ -z "$GETH_BASE_URL" ]; then
 fi
 
 function inc_status() {
-  echo " DONE!"
+  echo " done"
   STATUS=$((STATUS + 1))
   if [ $STATUS -lt ${#STATUSES[@]} ]; then
     echo -n "Waiting for ${STATUSES[$STATUS]}..."

app/data/container-build/cerc-foundry/build.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+# Build a local version of the foundry-rs/foundry image
+docker build -t cerc/foundry:local -f ${CERC_REPO_BASE_DIR}/foundry/Dockerfile-debian ${CERC_REPO_BASE_DIR}/foundry

app/data/container-build/cerc-laconic-registry-cli/Dockerfile

@@ -40,18 +40,20 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 # RUN su node -c "npm install -g <your-package-list-here>"
 
 # Configure the local npm registry
-RUN npm config set @lirewine:registry ${CERC_NPM_URL} \
-    && npm config set @cerc-io:registry ${CERC_NPM_URL} \
+RUN npm config set @cerc-io:registry ${CERC_NPM_URL} \
     && npm config set @lirewine:registry ${CERC_NPM_URL} \
     && npm config set -- ${CERC_NPM_URL}:_authToken ${CERC_NPM_AUTH_TOKEN}
 
 # TODO: the image at this point could be made a base image for several different CLI images
 # that install different Node-based CLI commands
 
-# DEBUG, remove
-RUN yarn info @cerc-io/laconic-registry-cli
-
 # Globally install the cli package
 RUN yarn global add @cerc-io/laconic-registry-cli
 
-ENTRYPOINT ["laconic"]
+# Add scripts
+RUN mkdir /scripts
+ENV PATH="${PATH}:/scripts"
+COPY ./import-key.sh /scripts
+
+# Default command sleeps forever so docker doesn't kill it
+CMD ["sh", "-c", "while :; do sleep 600; done"]

app/data/container-build/cerc-laconic-registry-cli/import-key.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+sed 's/REPLACE_WITH_MYKEY/'${1}'/' registry-cli-config-template.yml > config.yml

app/data/container-build/cerc-mobymask-ui/Dockerfile

@@ -0,0 +1,12 @@
+FROM node:18.15.0-alpine3.16
+
+RUN apk --update --no-cache add make git
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Building mobymask-ui" && \
+    npm install
+
+WORKDIR /app

app/data/container-build/cerc-mobymask-ui/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Build cerc/mobymask-ui
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/mobymask-ui:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/mobymask-ui

app/data/container-build/cerc-react-peer/Dockerfile

@@ -0,0 +1,12 @@
+FROM node:18.15.0-alpine3.16
+
+RUN apk --update --no-cache add make git
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Building react-peer" && \
+    yarn && yarn workspace @cerc-io/react-peer build
+
+WORKDIR /app

app/data/container-build/cerc-react-peer/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Build cerc/react-peer
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/react-peer:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/react-peer

app/data/container-build/cerc-watcher-mobymask-v2/Dockerfile

@@ -0,0 +1,20 @@
+FROM ubuntu:22.04
+
+RUN apt-get update \
+  && apt-get install -y curl gnupg build-essential \
+  && curl --silent --location https://deb.nodesource.com/setup_18.x | bash - \
+  && apt-get update \
+  && apt-get install -y nodejs git \
+  && node -v
+
+RUN corepack enable \
+  && yarn --version
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Building watcher-ts" && \
+    yarn && yarn build
+
+WORKDIR /app/packages/mobymask-v2-watcher

app/data/container-build/cerc-watcher-mobymask-v2/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Build cerc/watcher-mobymask-v2
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/watcher-mobymask-v2:local -f ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/watcher-ts

app/data/container-build/default-build.sh

@@ -11,4 +11,4 @@ fi
 image_tag=$1
 build_dir=$2
 echo "Building ${image_tag} in ${build_dir}"
-docker build -t ${image_tag} ${build_dir}
+docker build -t ${image_tag} --build-arg CERC_HOST_UID=${CERC_HOST_UID} --build-arg CERC_HOST_GID=${CERC_HOST_GID} ${build_dir}

app/data/container-build/foundry-rs-foundry/build.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-# Build foundry-rs/foundry
-# HACK below : TARGETARCH needs to be derived from the host environment
-docker build -t foundry-rs/foundry:local ${CERC_REPO_BASE_DIR}/foundry

app/data/container-image-list.txt

@@ -1,4 +1,4 @@
-foundry-rs/foundry
+cerc/foundry
 cerc/test-contract
 cerc/eth-statediff-fill-service
 cerc/eth-statediff-service
@@ -18,6 +18,9 @@ cerc/watcher-erc20
 cerc/watcher-erc721
 cerc/watcher-uniswap-v3
 cerc/uniswap-v3-info
+cerc/watcher-mobymask-v2
+cerc/react-peer
+cerc/mobymask-ui
 cerc/test-container
 cerc/eth-probe
 cerc/builder-js

app/data/npm-package-list.txt

@@ -1,4 +1,5 @@
-gem
 laconic-sdk
-debug
 laconic-registry-cli
+gem
+debug
+crypto

app/data/pod-list.txt

@@ -14,8 +14,10 @@ watcher-mobymask
 watcher-erc20
 watcher-erc721
 watcher-uniswap-v3
+watcher-mobymask-v2
 test
 eth-probe
 keycloak
 tx-spammer
 kubo
+foundry

app/data/repository-list.txt

@@ -11,9 +11,14 @@ cerc-io/laconic-sdk
 cerc-io/laconic-registry-cli
 cerc-io/mobymask-watcher
 cerc-io/watcher-ts
+cerc-io/react-peer
+cerc-io/mobymask-ui
 vulcanize/uniswap-watcher-ts
 vulcanize/uniswap-v3-info
 vulcanize/assemblyscript
 cerc-io/eth-probe
 cerc-io/tx-spammer
-foundry-rs/foundry
+dboreham/foundry
+lirewine/gem
+lirewine/debug
+lirewine/crypto

app/data/stacks/build-support/README.md

@@ -0,0 +1,59 @@
+# Build Support Stack
+
+## Instructions
+
+JS/TS/NPM builds need an npm registry to store intermediate package artifacts.
+This can be supplied by the user (e.g. using a hosted registry or even npmjs.com), or a local registry using gitea can be deployed by stack orchestrator.
+To use a user-supplied registry set these environment variables:
+
+`CERC_NPM_REGISTRY_URL` and 
+`CERC_NPM_AUTH_TOKEN`
+
+Leave `CERC_NPM_REGISTRY_URL` un-set to use the local gitea registry.
+
+### 1. Build support containers
+```
+$ laconic-so --stack build-support build-containers
+```
+Note that the scheme/gerbil builder container can take a while to build so if you aren't going to build scheme projects it can be skipped with:
+```
+$ laconic-so --stack build-support build-containers --exclude cerc/builder-gerbil
+```
+### 2. Deploy Gitea Package Registry
+
+```
+$ laconic-so --stack package-registry setup-repositories
+$ laconic-so --stack package-registry deploy up
+[+] Running 3/3
+ ⠿ Network laconic-aecc4a21d3a502b14522db97d427e850_gitea       Created                                                                                    0.0s
+ ⠿ Container laconic-aecc4a21d3a502b14522db97d427e850-db-1      Started                                                                                    1.2s
+ ⠿ Container laconic-aecc4a21d3a502b14522db97d427e850-server-1  Started                                                                                    1.9s
+New user 'gitea_admin' has been successfully created!
+This is your gitea access token: 84fe66a73698bf11edbdccd0a338236b7d1d5c45. Keep it safe and secure, it can not be fetched again from gitea.
+To use with laconic-so set this environment variable: export CERC_NPM_AUTH_TOKEN=3e493e77b3e83fe9e882f7e3a79dd4d5441c308b
+Created the organization cerc-io
+Gitea was configured to use host name: gitea.local, ensure that this resolves to localhost, e.g. with sudo vi /etc/hosts
+Success, gitea is properly initialized
+$
+```
+### 3. Configure the hostname gitea.local
+How to do this is OS-dependent but usually involves editing a `hosts` file. For example on Linux add this line to the file `/etc/hosts` (needs sudo):
+```
+127.0.0.1       gitea.local
+```
+Test with:
+```
+$ ping gitea.local
+PING gitea.local (127.0.0.1) 56(84) bytes of data.
+64 bytes from localhost (127.0.0.1): icmp_seq=1 ttl=64 time=0.147 ms
+64 bytes from localhost (127.0.0.1): icmp_seq=2 ttl=64 time=0.033 ms
+```
+Although not necessary in order to build and publish packages, you can now access the Gitea web interface at: [http://gitea.local:3000](http://gitea.local:3000) using these credentials: gitea_admin/admin1234 (Note: please properly secure Gitea if public internet access is allowed).
+
+Now npm packages can be built:
+### Build npm Packages
+Ensure that `CERC_NPM_AUTH_TOKEN` is set with the token printed above when the package-registry stack was deployed (the actual token value will be different than shown in this example):
+```
+$ export CERC_NPM_AUTH_TOKEN=84fe66a73698bf11edbdccd0a338236b7d1d5c45
+$ laconic-so build-npms --include laconic-sdk
+```

app/data/stacks/fixturenet-eth/README.md

@@ -20,27 +20,31 @@ This should create several container images in the local image registry:
 
 ## Deploy the stack
 ```
-$ laconic-so --stack fixturenet-eth deploy-system up
+$ laconic-so --stack fixturenet-eth deploy up
 ```
 
 ## Check status
 
 ```
-$ container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh
-Waiting for geth to generate DAG ..................................................... DONE!
-Waiting for beacon phase0 .... DONE!
-Waiting for beacon altair .... DONE!
-Waiting for beacon bellatrix pre-merge .... DONE!
-Waiting for beacon bellatrix merge .... DONE!
+$ laconic-so --stack fixturenet-eth deploy exec fixturenet-eth-bootnode-lighthouse /scripts/status-internal.sh
+Waiting for geth to generate DAG.... done
+Waiting for beacon phase0.... done
+Waiting for beacon altair.... done
+Waiting for beacon bellatrix pre-merge.... done
+Waiting for beacon bellatrix merge.... done
 
-$ docker ps -f 'name=laconic' --format 'table {{.Names}}\t{{.Ports}}'  | cut -d'-' -f3- | sort
-NAMES                                                                           PORTS
-fixturenet-eth-bootnode-geth-1         8545-8546/tcp, 30303/udp, 0.0.0.0:55847->9898/tcp, 0.0.0.0:55848->30303/tcp
-fixturenet-eth-bootnode-lighthouse-1
-fixturenet-eth-geth-1-1                8546/tcp, 30303/tcp, 30303/udp, 0.0.0.0:55851->8545/tcp
-fixturenet-eth-geth-2-1                8545-8546/tcp, 30303/tcp, 30303/udp
-fixturenet-eth-lighthouse-1-1          0.0.0.0:55858->8001/tcp
-fixturenet-eth-lighthouse-2-1
+$ laconic-so --stack fixturenet-eth deploy ps
+Running containers:
+id: c6538b60c0328dadfa2c5585c4d09674a6a13e6d712ff1cd82a26849e4e5679b, name: laconic-b12fa16e999821562937781f8ab0b1e8-fixturenet-eth-bootnode-geth-1, ports: 0.0.0.0:58909->30303/tcp, 0.0.0.0:58910->9898/tcp
+id: 5b70597a8211bc7e78d33e50486cb565a7f4a9ce581ce150b3bb450e342bdeda, name: laconic-b12fa16e999821562937781f8ab0b1e8-fixturenet-eth-bootnode-lighthouse-1, ports:
+id: 19ed78867b6c534d893835cdeb1e89a9ea553b8e8c02ab02468e4bd1563a340f, name: laconic-b12fa16e999821562937781f8ab0b1e8-fixturenet-eth-geth-1-1, ports: 0.0.0.0:58911->40000/tcp, 0.0.0.0:58912->6060/tcp, 0.0.0.0:58913->8545/tcp
+id: 8da0e30a1ce33122d8fd2225e4d26c7f30eb4bfbfa743f2af04d9db5d0bf7fa6, name: laconic-b12fa16e999821562937781f8ab0b1e8-fixturenet-eth-geth-2-1, ports:
+id: 387a42a14971034588ba9aeb9b9e2ca7fc0cc61b96f8fe8c2ab770c9d6fb1e0f, name: laconic-b12fa16e999821562937781f8ab0b1e8-fixturenet-eth-lighthouse-1-1, ports: 0.0.0.0:58917->8001/tcp
+id: de5115bf89087bae03b291664a73ffe3554fe23e79e4b8345e088b040d5580ac, name: laconic-b12fa16e999821562937781f8ab0b1e8-fixturenet-eth-lighthouse-2-1, ports:
+id: 2a7e5a0fb2be7fc9261a7b725a40818facbbe6d0cb2497d82c0e02de0a8e959b, name: laconic-b12fa16e999821562937781f8ab0b1e8-foundry-1, ports:
+
+$ laconic-so --stack fixturenet-eth deploy exec foundry "cast block-number"
+3
 ```
 
 ## Additional pieces
@@ -69,11 +73,11 @@ $ laconic-so deploy-system --include db,fixturenet-eth,ipld-eth-server,ipld-eth-
 # Status
 
 $ container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh
-Waiting for geth to generate DAG.... DONE!
-Waiting for beacon phase0.... DONE!
-Waiting for beacon altair.... DONE!
-Waiting for beacon bellatrix pre-merge.... DONE!
-Waiting for beacon bellatrix merge.... DONE!
+Waiting for geth to generate DAG.... done
+Waiting for beacon phase0.... done
+Waiting for beacon altair.... done
+Waiting for beacon bellatrix pre-merge.... done
+Waiting for beacon bellatrix merge.... done
 
 $ docker ps -f 'name=laconic' --format 'table {{.Names}}\t{{.Ports}}'  | cut -d'-' -f3- | sort
 NAMES                                                                           PORTS

app/data/stacks/fixturenet-eth/stack.yml

@@ -4,10 +4,13 @@ decription: "Ethereum Fixturenet"
 repos:
   - cerc-io/go-ethereum
   - cerc-io/tx-spammer
+  - dboreham/foundry
 containers:
   - cerc/go-ethereum
   - cerc/lighthouse
   - cerc/fixturenet-eth-geth
   - cerc/fixturenet-eth-lighthouse
+  - cerc/foundry
 pods:
   - fixturenet-eth
+  - foundry

app/data/stacks/fixturenet-laconicd/README.md

@@ -1,18 +1,48 @@
 # Laconicd Fixturenet
 
-Instructions for deploying a local Laconic blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator#user-mode)):
+Instructions for deploying a local Laconic blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator.
 
-## Clone required repositories
+## 1. Install Laconic Stack Orchestrator
+Installation is covered in detail [here](https://github.com/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as:
 ```
-$ laconic-so setup-repositories --include cerc-io/laconicd,cerc-io/laconic-sdk,cerc-io/laconic-registry-cli
+$ mkdir my-working-dir
+$ cd my-working-dir
+$ curl -L -o ./laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+$ chmod +x ./laconic-so
+$ export PATH=$PATH:$(pwd)  # Or move laconic-so to ~/bin or your favorite on-path directory
 ```
-## Build the laconicd container
+## 2. Prepare the local build environment
+Note that this step needs only to be done once on a new machine. 
+Detailed instructions can be found [here](../build-support/README.md). For the impatient run these commands:
 ```
-$ laconic-so build-containers --include cerc/laconicd
+$ laconic-so --stack build-support build-containers --exclude cerc/builder-gerbil
+$ laconic-so --stack package-registry setup-repositories
+$ laconic-so --stack package-registry deploy-system up
 ```
-This should create a container with tag `cerc/laconicd` in the local image registry.
-## Deploy the stack
+Then add the localhost alias `gitea.local` and set `CERC_NPM_AUTH_TOKEN` to the token printed when the package-registry stack was deployed above:
 ```
-$ laconic-so deploy-system --include fixturenet-laconicd up
+$ sudo vi /etc/hosts
+$ export CERC_NPM_AUTH_TOKEN=<my-token>
+```
+
+## 3. Clone required repositories
+```
+$ laconic-so --stack fixturenet-laconicd setup-repositories
+```
+## 4. Build the stack's packages and containers
+```
+$ laconic-so --stack fixturenet-laconicd build-npms
+$ laconic-so --stack fixturenet-laconicd build-containers
+```
+## 5. Deploy the stack
+```
+$ laconic-so --stack fixturenet-laconicd deploy up
+```
+Correct operation should be verified by checking the laconicd container's logs with:
+```
+$ laconic-so --stack fixturenet-laconicd deploy logs
+```
+## 6. Test with the Registry CLI
+```
+$ laconic-so --stack fixturenet-laconicd deploy exec cli "laconic cns status"
 ```
-Correct operation should be verified by checking the laconicd container's log.

app/data/stacks/fixturenet-laconicd/stack.yml

@@ -13,3 +13,6 @@ containers:
   - cerc/laconic-registry-cli
 pods:
   - fixturenet-laconicd
+config:
+  cli:
+    key: laconicd.mykey

app/data/stacks/mobymask-v2/README.md

@@ -0,0 +1,61 @@
+# MobyMask v2 watcher
+
+Instructions to deploy MobyMask v2 watcher stack using [laconic-stack-orchestrator](/README.md#install)
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack mobymask-v2 setup-repositories
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack mobymask-v2 build-containers
+```
+
+This should create the required docker images in the local image registry.
+
+Deploy the stack:
+
+```bash
+laconic-so --stack mobymask-v2 deploy-system up
+```
+
+## Tests
+
+Find the watcher container's id:
+
+```bash
+docker ps | grep "cerc/watcher-mobymask-v2:local"
+```
+
+Example output
+
+```
+8b38e9a64d7e   cerc/watcher-mobymask-v2:local   "sh -c 'yarn server'"    35 seconds ago   Up 14 seconds (health: starting)   0.0.0.0:3001->3001/tcp, 0.0.0.0:9001->9001/tcp, 0.0.0.0:9090->9090/tcp   laconic-aeb84676de2b0a7671ae90d537fc7d26-mobymask-watcher-server-1
+```
+
+In above output the container ID is `8b38e9a64d7e`
+
+Export it for later use:
+
+```bash
+export CONTAINER_ID=<CONTAINER_ID>
+```
+
+Run the peer tests:
+
+```bash
+docker exec -w /app/packages/peer $CONTAINER_ID yarn test
+```
+
+## Clean up
+
+To stop all the services running in background run:
+
+```bash
+laconic-so --stack mobymask-v2 deploy-system down
+```

app/data/stacks/mobymask-v2/stack.yml

@@ -0,0 +1,11 @@
+version: "1.0"
+name: mobymask-v2
+repos:
+  - cerc-io/watcher-ts
+  - cerc-io/react-peer
+  - cerc-io/mobymask-ui
+containers:
+  - cerc/watcher-mobymask-v2
+  - cerc/mobymask-ui
+pods:
+  - watcher-mobymask-v2

app/data/stacks/package-registry/README.md

@@ -0,0 +1,5 @@
+# Package Registry Stack
+
+The Package Registry Stack supports a build environment that requires a package registry (initially for NPM packages only).
+
+Setup instructions can be found [here](../build-support/README.md).

app/data/version.txt

@@ -1,2 +1,2 @@
 # This file should be re-generated running: scripts/update-version-file.sh script
-v1.0.20-6c1bedc
+v1.0.29-5fca9bd

app/deploy_system.py

@@ -19,9 +19,10 @@ import hashlib
 import copy
 import os
 import sys
+from dataclasses import dataclass
 from decouple import config
 import subprocess
-from python_on_whales import DockerClient
+from python_on_whales import DockerClient, DockerException
 import click
 import importlib.resources
 from pathlib import Path
@@ -56,15 +57,19 @@ def command(ctx, include, exclude, cluster, command, extra_args):
 
     if not dry_run:
         if command == "up":
-            if debug:
-                os.environ["CERC_SCRIPT_DEBUG"] = "true"
+            container_exec_env = _make_runtime_env(ctx.obj)
+            for attr, value in container_exec_env.items():
+                os.environ[attr] = value
             if verbose:
-                print(f"Running compose up for extra_args: {extra_args_list}")
+                print(f"Running compose up with container_exec_env: {container_exec_env}, extra_args: {extra_args_list}")
             for pre_start_command in cluster_context.pre_start_commands:
                 _run_command(ctx.obj, cluster_context.cluster, pre_start_command)
             docker.compose.up(detach=True, services=extra_args_list)
             for post_start_command in cluster_context.post_start_commands:
                 _run_command(ctx.obj, cluster_context.cluster, post_start_command)
+
+            _orchestrate_cluster_config(ctx.obj, cluster_context.config, docker, container_exec_env)
+
         elif command == "down":
             if verbose:
                 print("Running compose down")
@@ -74,13 +79,14 @@ def command(ctx, include, exclude, cluster, command, extra_args):
                 print("Usage: exec <service> <cmd>")
                 sys.exit(1)
             service_name = extra_args_list[0]
-            command_to_exec = extra_args_list[1:]
-            container_exec_env = {
-                "CERC_SCRIPT_DEBUG": "true"
-            } if debug else {}
+            command_to_exec = ["sh", "-c"] + extra_args_list[1:]
+            container_exec_env = _make_runtime_env(ctx.obj)
             if verbose:
                 print(f"Running compose exec {service_name} {command_to_exec}")
-            docker.compose.execute(service_name, command_to_exec, envs=container_exec_env)
+            try:
+                docker.compose.execute(service_name, command_to_exec, envs=container_exec_env)
+            except DockerException as error:
+                print(f"container command returned error exit status")
         elif command == "port":
             if extra_args_list is None or len(extra_args_list) < 2:
                 print("Usage: port <service> <exposed-port>")
@@ -115,7 +121,8 @@ def command(ctx, include, exclude, cluster, command, extra_args):
         elif command == "logs":
             if verbose:
                 print("Running compose logs")
-            docker.compose.logs()
+            logs_output = docker.compose.logs(services=extra_args_list if extra_args_list is not None else [])
+            print(logs_output)
 
 
 def get_stack_status(ctx, stack):
@@ -139,6 +146,15 @@ def get_stack_status(ctx, stack):
         False
 
 
+def _make_runtime_env(ctx):
+    container_exec_env = {
+        "CERC_HOST_UID": f"{os.getuid()}",
+        "CERC_HOST_GID": f"{os.getgid()}"
+    }
+    container_exec_env.update({"CERC_SCRIPT_DEBUG": "true"} if ctx.debug else {})
+    return container_exec_env
+
+
 def _make_cluster_context(ctx, include, exclude, cluster):
 
     if ctx.local_stack:
@@ -151,10 +167,13 @@ def _make_cluster_context(ctx, include, exclude, cluster):
     compose_dir = Path(__file__).absolute().parent.joinpath("data", "compose")
 
     if cluster is None:
-        # Create default unique, stable cluster name from confile file path
+        # Create default unique, stable cluster name from confile file path and stack name if provided
         # TODO: change this to the config file path
         path = os.path.realpath(sys.argv[0])
-        hash = hashlib.md5(path.encode()).hexdigest()
+        unique_cluster_descriptor = f"{path},{ctx.stack},{include},{exclude}"
+        if ctx.debug:
+            print(f"pre-hash descriptor: {unique_cluster_descriptor}")
+        hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()
         cluster = f"laconic-{hash}"
         if ctx.verbose:
             print(f"Using cluster name: {cluster}")
@@ -169,8 +188,10 @@ def _make_cluster_context(ctx, include, exclude, cluster):
         stack_config = get_parsed_stack_config(ctx.stack)
         # TODO: syntax check the input here
         pods_in_scope = stack_config['pods']
+        cluster_config = stack_config['config'] if 'config' in stack_config else None
     else:
         pods_in_scope = all_pods
+        cluster_config = None
 
     # Convert all pod definitions to v1.1 format
     pods_in_scope = _convert_to_new_format(pods_in_scope)
@@ -207,15 +228,16 @@ def _make_cluster_context(ctx, include, exclude, cluster):
     if ctx.verbose:
         print(f"files: {compose_files}")
 
-    return cluster_context(cluster, compose_files, pre_start_commands, post_start_commands)
+    return cluster_context(cluster, compose_files, pre_start_commands, post_start_commands, cluster_config)
 
 
 class cluster_context:
-    def __init__(self, cluster, compose_files, pre_start_commands, post_start_commands) -> None:
+    def __init__(self, cluster, compose_files, pre_start_commands, post_start_commands, config) -> None:
         self.cluster = cluster
         self.compose_files = compose_files
         self.pre_start_commands = pre_start_commands
         self.post_start_commands = post_start_commands
+        self.config = config
 
 
 def _convert_to_new_format(old_pod_array):
@@ -246,3 +268,45 @@ def _run_command(ctx, cluster_name, command):
     if command_result.returncode != 0:
         print(f"FATAL Error running command: {command}")
         sys.exit(1)
+
+
+def _orchestrate_cluster_config(ctx, cluster_config, docker, container_exec_env):
+
+    @dataclass
+    class ConfigDirective:
+        source_container: str
+        source_variable: str
+        destination_container: str
+        destination_variable: str
+
+    if cluster_config is not None:
+        for container in cluster_config:
+            container_config = cluster_config[container]
+            if ctx.verbose:
+                print(f"{container} config: {container_config}")
+            for directive in container_config:
+                pd = ConfigDirective(
+                    container_config[directive].split(".")[0],
+                    container_config[directive].split(".")[1],
+                    container,
+                    directive
+                )
+                if ctx.verbose:
+                    print(f"Setting {pd.destination_container}.{pd.destination_variable} = {pd.source_container}.{pd.source_variable}")
+                # TODO: fix the script paths so they're consistent between containers
+                source_value = docker.compose.execute(pd.source_container,
+                                                      ["sh", "-c",
+                                                          f"sh /docker-entrypoint-scripts.d/export-{pd.source_variable}.sh"],
+                                                      tty=False,
+                                                      envs=container_exec_env)
+                # TODO: handle the case that the value is not yet available
+                if ctx.debug:
+                    print(f"fetched source value: {source_value}")
+                destination_output = docker.compose.execute(pd.destination_container,
+                                                            ["sh", "-c",
+                                                             f"sh /scripts/import-{pd.destination_variable}.sh {source_value}"],
+                                                            tty=False,
+                                                            envs=container_exec_env)
+                if ctx.debug:
+                    print(f"destination output: {destination_output}")
+                # TODO: detect errors here

cli.py

@@ -55,5 +55,6 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err
 cli.add_command(setup_repositories.command, "setup-repositories")
 cli.add_command(build_containers.command, "build-containers")
 cli.add_command(build_npms.command, "build-npms")
+cli.add_command(deploy_system.command, "deploy")  # deploy is an alias for deploy-system
 cli.add_command(deploy_system.command, "deploy-system")
 cli.add_command(version.command, "version")

tests/smoke-test/run-smoke-test.sh

@@ -24,4 +24,11 @@ $TEST_TARGET_SO --stack test deploy-system up
 # TODO: test that we can use the deployed container somehow
 # Clean up
 $TEST_TARGET_SO --stack test deploy-system down
+# Run same test but not using the stack definition
+# Test building the a stack container
+$TEST_TARGET_SO build-containers --include cerc/test-container
+# Deploy the test container
+$TEST_TARGET_SO deploy-system --include test up
+# Clean up
+$TEST_TARGET_SO deploy-system --include test down
 echo "Test passed"