Merge pull request #197 from cerc-io/dboreham/uid-gid-to-deploy

Pass uid and gid to compose

Former-commit-id: 900917b548

app/build_containers.py

@@ -83,7 +83,10 @@ def command(ctx, include, exclude):
     container_build_env = {
         "CERC_NPM_URL": "http://gitea.local:3000/api/packages/cerc-io/npm/",
         "CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default="<token-not-supplied>"),
-        "CERC_REPO_BASE_DIR": dev_root_path
+        "CERC_REPO_BASE_DIR": dev_root_path,
+        "CERC_HOST_UID": f"{os.getuid()}",
+        "CERC_HOST_GID": f"{os.getgid()}",
+        "DOCKER_BUILDKIT": "0"
     }
 
     def process_container(container):
@@ -106,7 +109,7 @@ def command(ctx, include, exclude):
             build_command = os.path.join(container_build_dir, "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}"
         if not dry_run:
             if verbose:
-                print(f"Executing: {build_command}")
+                print(f"Executing: {build_command} with environment: {container_build_env}")
             build_result = subprocess.run(build_command, shell=True, env=container_build_env)
             if verbose:
                 print(f"Return code is: {build_result.returncode}")

app/build_npms.py

@@ -116,7 +116,11 @@ def command(ctx, include, exclude):
         if not dry_run:
             if verbose:
                 print(f"Executing: {build_command}")
-            envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token} | ({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
+            # Originally we used the PEP 584 merge operator:
+            # envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token} | ({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
+            # but that isn't available in Python 3.8 (default in Ubuntu 20) so for now we use dict.update:
+            envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token}
+            envs.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
             try:
                 docker.run(builder_js_image_name,
                            remove=True,

app/data/container-build/cerc-builder-js/Dockerfile

@@ -1,14 +1,30 @@
 # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
+# Which depends on: https://github.com/nodejs/docker-node/blob/main/Dockerfile-debian.template
 # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
 ARG VARIANT=16-bullseye
 FROM node:${VARIANT}
 
+# Set these args to change the uid/gid for the base container's "node" user to match that of the host user (so bind mounts work as expected).
+ARG CERC_HOST_UID=1000
+ARG CERC_HOST_GID=1000
+# Make these values available at runtime to allow a consistency check.
+ENV HOST_UID=${CERC_HOST_UID}
+ENV HOST_GID=${CERC_HOST_GID}
+
 ARG USERNAME=node
 ARG NPM_GLOBAL=/usr/local/share/npm-global
 
 # Add NPM global to PATH.
 ENV PATH=${NPM_GLOBAL}/bin:${PATH}
 
+RUN \
+    if [ ${CERC_HOST_GID} -ne 1000 ] ; then \
+        groupmod -g ${CERC_HOST_GID} ${USERNAME} ; \
+    fi \
+    && if [ ${CERC_HOST_UID} -ne 1000 ] ; then \
+        usermod -u ${CERC_HOST_UID} -g ${CERC_HOST_GID} ${USERNAME} && chown ${CERC_HOST_UID}:${CERC_HOST_GID} /home/${USERNAME} ; \
+    fi
+
 RUN \
     # Configure global npm install location, use group to adapt to UID/GID changes
     if ! cat /etc/group | grep -e "^npm:" > /dev/null 2>&1; then groupadd -r npm; fi \
@@ -39,6 +55,7 @@ RUN mkdir /scripts
 COPY build-npm-package.sh /scripts
 COPY yarn-local-registry-fixup.sh /scripts
 COPY build-npm-package-local-dependencies.sh /scripts
+COPY check-uid.sh /scripts
 ENV PATH="${PATH}:/scripts"
 
 COPY entrypoint.sh .

app/data/container-build/cerc-builder-js/check-uid.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Make the container usable for uid/gid != 1000
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+current_uid=$(id -u)
+current_gid=$(id -g)
+# Don't check if running as root
+if [[ ${current_uid} == 0 ]]; then
+    exit 0
+fi
+# Check the current uid/gid vs the uid/gid used to build the container.
+# We do this because both bind mounts and npm tooling require the uid/gid to match.
+if [[ ${current_gid} != ${HOST_GID} ]]; then
+    echo "Warning: running with gid: ${current_gid} which is not the gid for which this container was built (${HOST_GID})"
+    exit 0
+fi
+if [[ ${current_uid} != ${HOST_UID} ]]; then
+    echo "Warning: running with gid: ${current_uid} which is not the uid for which this container was built (${HOST_UID})"
+    exit 0
+fi

app/data/container-build/cerc-builder-js/entrypoint.sh

@@ -1,2 +1,3 @@
 #!/bin/sh
+/scripts/check-uid.sh
 exec "$@"

app/data/container-build/default-build.sh

@@ -11,4 +11,4 @@ fi
 image_tag=$1
 build_dir=$2
 echo "Building ${image_tag} in ${build_dir}"
-docker build -t ${image_tag} ${build_dir}
+docker build -t ${image_tag} --build-arg CERC_HOST_UID=${CERC_HOST_UID} --build-arg CERC_HOST_GID=${CERC_HOST_GID} ${build_dir}

app/data/stacks/build-support/README.md

@@ -0,0 +1,30 @@
+# Build Support Stack
+
+## Instructions
+
+JS/TS/NPM builds need an npm registry to store intermediate package artifacts.
+This can be supplied by the user (e.g. using a hosted registry or even npmjs.com), or a local registry using gitea can be deployed by stack orchestrator.
+To use a user-supplied registry set these environment variables:
+
+`CERC_NPM_REGISTRY_URL` and 
+`CERC_NPM_AUTH_TOKEN`
+
+Leave `CERC_NPM_REGISTRY_URL` un-set to use the local gitea registry.
+
+### Build support containers
+```
+$ laconic-so --stack build-support build-containers
+```
+### Deploy Gitea Package Registry
+
+```
+$ laconic-so --stack package-registry setup-repositories
+$ laconic-so --stack package-registry deploy-system up
+This is your gitea access token: 84fe66a73698bf11edbdccd0a338236b7d1d5c45. Keep it safe and secure, it can not be fetched again from gitea.
+$ export CERC_NPM_AUTH_TOKEN=84fe66a73698bf11edbdccd0a338236b7d1d5c45
+```
+Now npm packages can be built:
+### Build npm Packages
+```
+$ laconic-so build-npms --include laconic-sdk
+```

app/data/version.txt

@@ -1,2 +1,2 @@
 # This file should be re-generated running: scripts/update-version-file.sh script
-v1.0.20-6c1bedc
+v1.0.22-4fdb442

app/deploy_system.py

@@ -76,8 +76,10 @@ def command(ctx, include, exclude, cluster, command, extra_args):
             service_name = extra_args_list[0]
             command_to_exec = extra_args_list[1:]
             container_exec_env = {
-                "CERC_SCRIPT_DEBUG": "true"
-            } if debug else {}
+                "CERC_HOST_UID": f"{os.getuid()}",
+                "CERC_HOST_GID": f"{os.getgid()}"
+            }
+            container_exec_env.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
             if verbose:
                 print(f"Running compose exec {service_name} {command_to_exec}")
             docker.compose.execute(service_name, command_to_exec, envs=container_exec_env)
@@ -151,10 +153,13 @@ def _make_cluster_context(ctx, include, exclude, cluster):
     compose_dir = Path(__file__).absolute().parent.joinpath("data", "compose")
 
     if cluster is None:
-        # Create default unique, stable cluster name from confile file path
+        # Create default unique, stable cluster name from confile file path and stack name if provided
         # TODO: change this to the config file path
         path = os.path.realpath(sys.argv[0])
-        hash = hashlib.md5(path.encode()).hexdigest()
+        unique_cluster_descriptor = f"{path},{ctx.stack},{include},{exclude}"
+        if ctx.debug:
+            print(f"pre-hash descriptor: {unique_cluster_descriptor}")
+        hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()
         cluster = f"laconic-{hash}"
         if ctx.verbose:
             print(f"Using cluster name: {cluster}")

cli.py

@@ -55,5 +55,6 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err
 cli.add_command(setup_repositories.command, "setup-repositories")
 cli.add_command(build_containers.command, "build-containers")
 cli.add_command(build_npms.command, "build-npms")
+cli.add_command(deploy_system.command, "deploy")  # deploy is an alias for deploy-system
 cli.add_command(deploy_system.command, "deploy-system")
 cli.add_command(version.command, "version")