Rename app -> stack_orchestrator (#625)

2023-11-07 00:06:55 -07:00
parent e989368793
commit 4456e70c93
483 changed files with 82 additions and 81 deletions
@@ -0,0 +1,72 @@
+# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
+# Which depends on: https://github.com/nodejs/docker-node/blob/main/Dockerfile-debian.template
+# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
+ARG VARIANT=18-bullseye
+FROM node:${VARIANT}
+
+# Set these args to change the uid/gid for the base container's "node" user to match that of the host user (so bind mounts work as expected).
+ARG CERC_HOST_UID=1000
+ARG CERC_HOST_GID=1000
+# Make these values available at runtime to allow a consistency check.
+ENV HOST_UID=${CERC_HOST_UID}
+ENV HOST_GID=${CERC_HOST_GID}
+
+ARG USERNAME=node
+ARG NPM_GLOBAL=/usr/local/share/npm-global
+
+# Add NPM global to PATH.
+ENV PATH=${NPM_GLOBAL}/bin:${PATH}
+
+SHELL ["/bin/bash", "-c"]
+
+RUN \
+    # Don't switch container uid/gid if the host uid/gid is 1000 (which means it's already correct),
+    # or root (which won't work anyway) or <= 100 (which also won't work).
+    if [[ ${CERC_HOST_GID} -ne 1000 && ${CERC_HOST_GID} -ne 0 && ${CERC_HOST_GID} -gt 100 ]]; then \
+        groupmod -g ${CERC_HOST_GID} ${USERNAME}; \
+    fi \
+    && if [[ ${CERC_HOST_UID} -ne 1000 && ${CERC_HOST_UID} -ne 0 && ${CERC_HOST_UID} -gt 100 ]]; then \
+        usermod -u ${CERC_HOST_UID} -g ${CERC_HOST_GID} ${USERNAME} && chown ${CERC_HOST_UID}:${CERC_HOST_GID} /home/${USERNAME}; \
+    fi
+
+# Prevents npm from printing version warnings
+ENV NPM_CONFIG_UPDATE_NOTIFIER=false
+
+RUN \
+    # Configure global npm install location, use group to adapt to UID/GID changes
+    if ! cat /etc/group | grep -e "^npm:" > /dev/null 2>&1; then groupadd -r npm; fi \
+    && usermod -a -G npm ${USERNAME} \
+    && umask 0002 \
+    && mkdir -p ${NPM_GLOBAL} \
+    && touch /usr/local/etc/npmrc \
+    && chown ${USERNAME}:npm ${NPM_GLOBAL} /usr/local/etc/npmrc \
+    && chmod g+s ${NPM_GLOBAL} \
+    && npm config -g set prefix ${NPM_GLOBAL} \
+    && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
+    # Install eslint
+    && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
+    && npm cache clean --force > /dev/null 2>&1
+
+# [Optional] Uncomment this section to install additional OS packages.
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -y install --no-install-recommends jq
+
+# [Optional] Uncomment if you want to install an additional version of node using nvm
+# ARG EXTRA_NODE_VERSION=10
+# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}"
+
+# [Optional] Uncomment if you want to install more global node modules
+# RUN su node -c "npm install -g <your-package-list-here>"
+
+RUN mkdir /scripts
+COPY build-npm-package.sh /scripts
+COPY yarn-local-registry-fixup.sh /scripts
+COPY build-npm-package-local-dependencies.sh /scripts
+COPY check-uid.sh /scripts
+ENV PATH="${PATH}:/scripts"
+
+COPY entrypoint.sh .
+ENTRYPOINT ["./entrypoint.sh"]
+# Placeholder CMD : generally this will be overridden at run time like :
+# docker run -it -v /home/builder/cerc/laconic-sdk:/workspace cerc/builder-js sh -c 'cd /workspace && yarn && yarn build'
+CMD node --version
@@ -0,0 +1,17 @@
+## JS/TS Package Builder
+
+This container is designed to be used as a simple "build runner" environment for building and publishing JS/TS projects
+using `yarn`.
+
+### Running a build
+
+As a temporary measure while the necessary functionality is being added to Stack Orchestrator,
+it is possible to build packages manually by invoking `docker run` , for example as follows:
+
+
+```
+$ docker run --rm -it --add-host gitea.local:host-gateway \
+  -v ${HOME}/cerc/laconic-registry-cli:/workspace cerc/builder-js  \
+  sh -c 'cd /workspace && CERC_NPM_AUTH_TOKEN=6613572a28ebebaee20ccd90064251fa8c2b94f6 \
+  build-npm-package-local-dependencies.sh http://gitea.local:3000/api/packages/cerc-io/npm/ 0.1.8'
+```
@@ -0,0 +1,37 @@
+#!/bin/bash
+# Usage: build-npm-package-local-dependencies.sh <registry-url> <publish-with-this-version>
+# Runs build-npm-package.sh after first fixing up yarn.lock to use a local
+# npm registry for all packages in a specific scope (currently @cerc-io and @lirewine)
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+if ! [[ $# -eq 1 || $# -eq 2 ]]; then
+    echo "Illegal number of parameters" >&2
+    exit 1
+fi
+if [[ -z "${CERC_NPM_AUTH_TOKEN}" ]]; then
+    echo "CERC_NPM_AUTH_TOKEN is not set" >&2
+    exit 1
+fi
+# Exit on error
+set -e
+local_npm_registry_url=$1
+package_publish_version=$2
+# If we need to handle an additional scope, add it to the list below:
+npm_scopes_to_handle=("@cerc-io" "@lirewine")
+for npm_scope_for_local in ${npm_scopes_to_handle[@]}
+do
+    # We need to configure the local registry
+    npm config set ${npm_scope_for_local}:registry ${local_npm_registry_url}
+    npm config set -- ${local_npm_registry_url}:_authToken ${CERC_NPM_AUTH_TOKEN}
+    # Find the set of dependencies from the specified scope
+    mapfile -t dependencies_from_scope < <(cat package.json | jq -r '.dependencies | with_entries(if (.key|test("^'${npm_scope_for_local}'/.*$")) then ( {key: .key, value: .value } ) else empty end ) | keys[]')
+    echo "Fixing up dependencies in scope ${npm_scope_for_local}"
+    for package in "${dependencies_from_scope[@]}"
+    do
+        echo "Fixing up package ${package}"
+        yarn-local-registry-fixup.sh $package ${local_npm_registry_url}
+    done
+done
+echo "Running build"
+build-npm-package.sh ${local_npm_registry_url} ${package_publish_version}
@@ -0,0 +1,46 @@
+#!/bin/bash
+# Usage: build-npm-package.sh <registry-url> <publish-with-this-version>
+# Note: supply the registry auth token in CERC_NPM_AUTH_TOKEN
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+if ! [[ $# -eq 1 || $# -eq 2 ]]; then
+    echo "Illegal number of parameters" >&2
+    exit 1
+fi
+if [[ -z "${CERC_NPM_AUTH_TOKEN}" ]]; then
+    echo "CERC_NPM_AUTH_TOKEN is not set" >&2
+    exit 1
+fi
+if [[ $# -eq 2 ]]; then
+    package_publish_version=$2
+else
+    package_publish_version=$( cat package.json | jq -r .version )
+fi
+# Exit on error
+set -e
+# Get the name of this package from package.json since we weren't passed that
+package_name=$( cat package.json | jq -r .name )
+local_npm_registry_url=$1
+npm config set @cerc-io:registry ${local_npm_registry_url}
+npm config set @lirewine:registry ${local_npm_registry_url}
+# Workaround bug in npm unpublish where it needs the url to be of the form //<foo> and not http://<foo>
+local_npm_registry_url_fixed=$( echo ${local_npm_registry_url} | sed -e 's/^http[s]\{0,1\}://')
+npm config set -- ${local_npm_registry_url_fixed}:_authToken ${CERC_NPM_AUTH_TOKEN}
+# First check if the version of this package we're trying to build already exists in the registry
+package_exists=$( yarn info --json ${package_name}@${package_publish_version} 2>/dev/null | jq -r .data.dist.tarball )
+if [[ ! -z "$package_exists" && "$package_exists" != "null" ]]; then
+    echo "${package_publish_version} of ${package_name} already exists in the registry"
+    if [[ ${CERC_FORCE_REBUILD} == "true" ]]; then
+        # Attempt to unpublish the existing package
+        echo "NOTE: unpublishing existing package version since force rebuild is enabled"
+        npm unpublish --force ${package_name}@${package_publish_version}
+    else
+        echo "skipping build since target version already exists"
+        exit 0
+    fi
+fi
+echo "Build and publish ${package_name} version ${package_publish_version}"
+yarn install
+yarn build
+yarn publish --non-interactive  --new-version ${package_publish_version} --no-git-tag-version
@@ -0,0 +1,21 @@
+#!/bin/bash
+# Make the container usable for uid/gid != 1000
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+current_uid=$(id -u)
+current_gid=$(id -g)
+# Don't check if running as root
+if [[ ${current_uid} == 0 ]]; then
+    exit 0
+fi
+# Check the current uid/gid vs the uid/gid used to build the container.
+# We do this because both bind mounts and npm tooling require the uid/gid to match.
+if [[ ${current_gid} != ${HOST_GID} ]]; then
+    echo "Warning: running with gid: ${current_gid} which is not the gid for which this container was built (${HOST_GID})"
+    exit 0
+fi
+if [[ ${current_uid} != ${HOST_UID} ]]; then
+    echo "Warning: running with gid: ${current_uid} which is not the uid for which this container was built (${HOST_UID})"
+    exit 0
+fi
@@ -0,0 +1,3 @@
+#!/bin/sh
+/scripts/check-uid.sh
+exec "$@"
@@ -0,0 +1,58 @@
+#!/bin/bash
+# Usage: yarn-local-registry-fixup.sh <package-to-fix> <registry-url>
+# Assumes package.json and yarn.lock are in the cwd
+# The purpose of this script is to take a project cloned from git
+# and "fixup" its yarn.lock file such that specified dependency
+# will be fetched from a registry other than the one used when
+# yarn.lock was generated. It updates all checksums using data
+# from the "new" registry (because due to embedded timestamps etc
+# the same source code re-built later will not have the same checksum).
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+if [[ $# -ne 2 ]]; then
+    echo "Illegal number of parameters" >&2
+    exit 1
+fi
+# Exit on error
+set -e
+target_package=$1
+local_npm_registry_url=$2
+# Extract the actual version pinned in yarn.lock
+# See: https://stackoverflow.com/questions/60454251/how-to-know-the-version-of-currently-installed-package-from-yarn-lock
+versioned_target_package=$(yarn list --pattern ${target_package} --depth=0 --json --non-interactive --no-progress | jq -r '.data.trees[].name')
+# Use yarn info to get URL checksums etc from the new registry
+yarn_info_output=$(yarn info --json $versioned_target_package 2>/dev/null)
+# First check if the target version actually exists. 
+# If it doesn't exist there will be no .data.dist.tarball element,
+# and jq will output the string "null"
+package_tarball=$(echo $yarn_info_output | jq -r .data.dist.tarball)
+if [[ "$yarn_info_output" == "" ||  $package_tarball == "null" ]]; then
+    echo "FATAL: Target package version ($versioned_target_package) not found (or bad npm auth token)" >&2
+    exit 1
+fi
+# Code below parses out the values we need
+# When running inside a container, the registry can return a URL with the wrong host name due to proxying
+# so we need to check if that has happened and fix the URL if so.
+if ! [[ "${package_tarball}" =~ ^${local_npm_registry_url}.* ]]; then
+    # HACK: I've hard-wired the host names below. Replace with proper implementation
+    # TODO: remove the hack when proven no longer necessary
+    package_tarball=$( echo ${package_tarball} | sed -e 's/localhost/gitea.local/g' )
+fi
+package_integrity=$(echo $yarn_info_output | jq -r .data.dist.integrity)
+package_shasum=$(echo $yarn_info_output | jq -r .data.dist.shasum)
+package_resolved=${package_tarball}#${package_shasum}
+# Some strings need to be escaped so they work when passed to sed later
+escaped_package_integrity=$(printf '%s\n' "$package_integrity" | sed -e 's/[\/&]/\\&/g')
+escaped_package_resolved=$(printf '%s\n' "$package_resolved" | sed -e 's/[\/&]/\\&/g')
+escaped_target_package=$(printf '%s\n' "$target_package" | sed -e 's/[\/&]/\\&/g')
+if [ -n "$CERC_SCRIPT_VERBOSE" ]; then
+    echo "Tarball: ${package_tarball}"
+    echo "Integrity: ${package_integrity}"
+    echo "Shasum: ${package_shasum}"
+    echo "Resolved: ${package_resolved}"
+fi
+# Use magic sed regex to replace the values in yarn.lock
+# Note: yarn.lock is not json so we can not use jq for this
+sed -i -e '/^\"'${escaped_target_package}'.*\":$/ , /^\".*$/ s/^\([[:space:]]\{1,\}resolved \).*$/\1'\"${escaped_package_resolved}\"'/' yarn.lock
+sed -i -e '/^\"'${escaped_target_package}'.*\":$/ , /^\".*$/ s/^\([[:space:]]\{1,\}integrity \).*$/\1'${escaped_package_integrity}'/' yarn.lock