diff --git a/stack_orchestrator/build/build_containers.py b/stack_orchestrator/build/build_containers.py index c97a974f..5b2748cc 100644 --- a/stack_orchestrator/build/build_containers.py +++ b/stack_orchestrator/build/build_containers.py @@ -33,6 +33,73 @@ from stack_orchestrator.base import get_npm_registry_url # TODO: find a place for this # epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)" +def make_container_build_env(dev_root_path: str, + container_build_dir: str, + debug: bool, + force_rebuild: bool, + extra_build_args: str): + container_build_env = { + "CERC_NPM_REGISTRY_URL": get_npm_registry_url(), + "CERC_GO_AUTH_TOKEN": config("CERC_GO_AUTH_TOKEN", default=""), + "CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default=""), + "CERC_REPO_BASE_DIR": dev_root_path, + "CERC_CONTAINER_BASE_DIR": container_build_dir, + "CERC_HOST_UID": f"{os.getuid()}", + "CERC_HOST_GID": f"{os.getgid()}", + "DOCKER_BUILDKIT": config("DOCKER_BUILDKIT", default="0") + } + container_build_env.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {}) + container_build_env.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {}) + container_build_env.update({"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {}) + docker_host_env = os.getenv("DOCKER_HOST") + if docker_host_env: + container_build_env.update({"DOCKER_HOST": docker_host_env}) + + return container_build_env + + +def process_container(container, + container_build_dir: str, + container_build_env: dict, + dev_root_path: str, + quiet: bool, + verbose: bool, + dry_run: bool, + continue_on_error: bool, + ): + if not quiet: + print(f"Building: {container}") + build_dir = os.path.join(container_build_dir, container.replace("/", "-")) + build_script_filename = os.path.join(build_dir, "build.sh") + if verbose: + print(f"Build script filename: {build_script_filename}") + if os.path.exists(build_script_filename): + build_command = build_script_filename + else: + if verbose: + print(f"No script file found: {build_script_filename}, using default build script") + repo_dir = container.split('/')[1] + # TODO: make this less of a hack -- should be specified in some metadata somewhere + # Check if we have a repo for this container. If not, set the context dir to the container-build subdir + repo_full_path = os.path.join(dev_root_path, repo_dir) + repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir + build_command = os.path.join(container_build_dir, + "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}" + if not dry_run: + if verbose: + print(f"Executing: {build_command} with environment: {container_build_env}") + build_result = subprocess.run(build_command, shell=True, env=container_build_env) + if verbose: + print(f"Return code is: {build_result.returncode}") + if build_result.returncode != 0: + print(f"Error running build for {container}") + if not continue_on_error: + print("FATAL Error: container build failed and --continue-on-error not set, exiting") + sys.exit(1) + else: + print("****** Container Build Error, continuing because --continue-on-error is set") + else: + print("Skipped") @click.command() @click.option('--include', help="only build these containers") @@ -83,61 +150,16 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args): if stack: print(f"Stack: {stack}") - # TODO: make this configurable - container_build_env = { - "CERC_NPM_REGISTRY_URL": get_npm_registry_url(), - "CERC_GO_AUTH_TOKEN": config("CERC_GO_AUTH_TOKEN", default=""), - "CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default=""), - "CERC_REPO_BASE_DIR": dev_root_path, - "CERC_CONTAINER_BASE_DIR": container_build_dir, - "CERC_HOST_UID": f"{os.getuid()}", - "CERC_HOST_GID": f"{os.getgid()}", - "DOCKER_BUILDKIT": config("DOCKER_BUILDKIT", default="0") - } - container_build_env.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {}) - container_build_env.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {}) - container_build_env.update({"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {}) - docker_host_env = os.getenv("DOCKER_HOST") - if docker_host_env: - container_build_env.update({"DOCKER_HOST": docker_host_env}) - - def process_container(container): - if not quiet: - print(f"Building: {container}") - build_dir = os.path.join(container_build_dir, container.replace("/", "-")) - build_script_filename = os.path.join(build_dir, "build.sh") - if verbose: - print(f"Build script filename: {build_script_filename}") - if os.path.exists(build_script_filename): - build_command = build_script_filename - else: - if verbose: - print(f"No script file found: {build_script_filename}, using default build script") - repo_dir = container.split('/')[1] - # TODO: make this less of a hack -- should be specified in some metadata somewhere - # Check if we have a repo for this container. If not, set the context dir to the container-build subdir - repo_full_path = os.path.join(dev_root_path, repo_dir) - repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir - build_command = os.path.join(container_build_dir, "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}" - if not dry_run: - if verbose: - print(f"Executing: {build_command} with environment: {container_build_env}") - build_result = subprocess.run(build_command, shell=True, env=container_build_env) - if verbose: - print(f"Return code is: {build_result.returncode}") - if build_result.returncode != 0: - print(f"Error running build for {container}") - if not continue_on_error: - print("FATAL Error: container build failed and --continue-on-error not set, exiting") - sys.exit(1) - else: - print("****** Container Build Error, continuing because --continue-on-error is set") - else: - print("Skipped") + container_build_env = make_container_build_env(dev_root_path, + container_build_dir, + debug, + force_rebuild, + extra_build_args) for container in containers_in_scope: if include_exclude_check(container, include, exclude): - process_container(container) + process_container(container, container_build_dir, container_build_env, + dev_root_path, quiet, verbose, dry_run, continue_on_error) else: if verbose: print(f"Excluding: {container}") diff --git a/stack_orchestrator/build/build_webapp.py b/stack_orchestrator/build/build_webapp.py new file mode 100644 index 00000000..00b248ad --- /dev/null +++ b/stack_orchestrator/build/build_webapp.py @@ -0,0 +1,73 @@ +# Copyright © 2022, 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Builds webapp containers + +# env vars: +# CERC_REPO_BASE_DIR defaults to ~/cerc + +# TODO: display the available list of containers; allow re-build of either all or specific containers + +import os +from decouple import config +import click +from pathlib import Path +from stack_orchestrator.build import build_containers + + +@click.command() +@click.option('--base-container', default="cerc/nextjs-base") +@click.option('--source-repo', help="directory containing the webapp to build", required=True) +@click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild") +@click.option("--extra-build-args", help="Supply extra arguments to build") +@click.pass_context +def command(ctx, base_container, source_repo, force_rebuild, extra_build_args): + '''build the specified webapp container''' + + quiet = ctx.obj.quiet + verbose = ctx.obj.verbose + dry_run = ctx.obj.dry_run + debug = ctx.obj.debug + local_stack = ctx.obj.local_stack + stack = ctx.obj.stack + continue_on_error = ctx.obj.continue_on_error + + # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure + container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build") + + if local_stack: + dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] + print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') + else: + dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) + + if not quiet: + print(f'Dev Root is: {dev_root_path}') + + container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug, + force_rebuild, extra_build_args) + + build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet, + verbose, dry_run, continue_on_error) + + container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo) + container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir, + base_container.replace("/", "-"), + "Dockerfile.webapp") + webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1] + container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local" + + build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet, + verbose, dry_run, continue_on_error) diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile new file mode 100644 index 00000000..147cec29 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile @@ -0,0 +1,55 @@ +# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile +# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster +ARG VARIANT=18-bullseye +FROM node:${VARIANT} + +ARG USERNAME=node +ARG NPM_GLOBAL=/usr/local/share/npm-global + +# Add NPM global to PATH. +ENV PATH=${NPM_GLOBAL}/bin:${PATH} +# Prevents npm from printing version warnings +ENV NPM_CONFIG_UPDATE_NOTIFIER=false + +RUN \ + # Configure global npm install location, use group to adapt to UID/GID changes + if ! cat /etc/group | grep -e "^npm:" > /dev/null 2>&1; then groupadd -r npm; fi \ + && usermod -a -G npm ${USERNAME} \ + && umask 0002 \ + && mkdir -p ${NPM_GLOBAL} \ + && touch /usr/local/etc/npmrc \ + && chown ${USERNAME}:npm ${NPM_GLOBAL} /usr/local/etc/npmrc \ + && chmod g+s ${NPM_GLOBAL} \ + && npm config -g set prefix ${NPM_GLOBAL} \ + && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \ + # Install eslint + && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \ + && npm cache clean --force > /dev/null 2>&1 + +# [Optional] Uncomment this section to install additional OS packages. +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends jq gettext-base + +# [Optional] Uncomment if you want to install an additional version of node using nvm +# ARG EXTRA_NODE_VERSION=10 +# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}" + +# We do this to get a yq binary from the published container, for the correct architecture we're building here +# COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq + +COPY /scripts /scripts + +# [Optional] Uncomment if you want to install more global node modules +# RUN su node -c "npm install -g " + +# RUN mkdir -p /config +# COPY ./config.yml /config + +# Install simple web server for now (use nginx perhaps later) +# RUN yarn global add http-server + +# Expose port for http +EXPOSE 3000 + +# Default command sleeps forever so docker doesn't kill it +CMD ["/scripts/start-serving-app.sh"] diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp new file mode 100644 index 00000000..f4b5d4d8 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp @@ -0,0 +1,5 @@ +FROM cerc/nextjs-base:local +WORKDIR /app +COPY . . +RUN rm -rf node_modules build .next* +RUN /scripts/build-app.sh /app diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh new file mode 100755 index 00000000..1525708d --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Build cerc/laconic-registry-cli + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +CERC_CONTAINER_BUILD_WORK_DIR=${CERC_CONTAINER_BUILD_WORK_DIR:-$SCRIPT_DIR} +CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/Dockerfile} +CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/nextjs-base:local} + +docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR \ No newline at end of file diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh new file mode 100755 index 00000000..ba1cd17d --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +WORK_DIR="${1:-./}" +SRC_DIR="${2:-.next}" +TRG_DIR="${3:-.next-r}" + +cd "${WORK_DIR}" || exit 1 + +rm -rf "$TRG_DIR" +mkdir -p "$TRG_DIR" +cp -rp "$SRC_DIR" "$TRG_DIR/" + +if [ -f ".env" ]; then + TMP_ENV=`mktemp` + declare -px > $TMP_ENV + set -a + source .env + source $TMP_ENV + set +a + rm -f $TMP_ENV +fi + +for f in $(find "$TRG_DIR" -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do + for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o '^"CERC_RUNTIME_ENV[^\"]+"$'); do + orig_name=$(echo -n "${e}" | sed 's/"//g') + cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g') + cur_val=$(echo -n "\$${cur_name}" | envsubst) + esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val") + echo "$cur_name=$cur_val" + sed -i "s/$orig_name/$esc_val/g" $f + done +done diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh new file mode 100755 index 00000000..9277abc6 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +WORK_DIR="${1:-/app}" + +cd "${WORK_DIR}" || exit 1 + +cp next.config.js next.config.dist + +npm i -g js-beautify +js-beautify next.config.dist > next.config.js + +npm install + +CONFIG_LINES=$(wc -l next.config.js | awk '{ print $1 }') +MOD_EXPORTS_LINE=$(grep -n 'module.exports' next.config.js | cut -d':' -f1) + +head -$(( ${MOD_EXPORTS_LINE} - 1 )) next.config.js > next.config.js.1 + +cat > next.config.js.2 < { + a[v] = \`"CERC_RUNTIME_ENV_\${v.split(/\./).pop()}"\`; + return a; + }, {}); +} catch { + // If .env-list.json cannot be loaded, we are probably running in dev mode, so use process.env instead. + envMap = Object.keys(process.env).reduce((a, v) => { + if (v.startsWith('CERC_')) { + a[\`process.env.\${v}\`] = JSON.stringify(process.env[v]); + } + return a; + }, {}); +} +EOF + +grep 'module.exports' next.config.js > next.config.js.3 + +cat > next.config.js.4 < { + config.plugins.push(new webpack.DefinePlugin(envMap)); + return config; + }, +EOF + +tail -$(( ${CONFIG_LINES} - ${MOD_EXPORTS_LINE} + 1 )) next.config.js | grep -v 'process\.env\.' > next.config.js.5 + +cat next.config.js.* | js-beautify > next.config.js +rm next.config.js.* + +"${SCRIPT_DIR}/find-env.sh" "$(pwd)" > .env-list.json + +npm run build +rm .env-list.json \ No newline at end of file diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh new file mode 100755 index 00000000..55679a06 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +WORK_DIR="${1:-./}" +TMPF=$(mktemp) + +cd "$WORK_DIR" || exit 1 + +for d in $(find . -maxdepth 1 -type d | grep -v '\./\.' | grep '/' | cut -d'/' -f2); do + egrep "/$d[/$]?" .gitignore >/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + continue + fi + + for f in $(find "$d" -regex ".*.[tj]sx?$" -type f); do + cat "$f" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o 'process.env.[A-Za-z0-9_]+' >> $TMPF + done +done + +cat $TMPF | sort -u | jq --raw-input . | jq --slurp . +rm -f $TMPF \ No newline at end of file diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh new file mode 100755 index 00000000..abe72935 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/app}" +cd "$CERC_WEBAPP_FILES_DIR" + +rm -rf .next-r +"$SCRIPT_DIR/apply-runtime-env.sh" "`pwd`" .next .next-r +npm start .next-r -p ${CERC_LISTEN_PORT:-3000} diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py index ca1914e6..0b0585e0 100644 --- a/stack_orchestrator/main.py +++ b/stack_orchestrator/main.py @@ -19,6 +19,7 @@ from stack_orchestrator.command_types import CommandOptions from stack_orchestrator.repos import setup_repositories from stack_orchestrator.build import build_containers from stack_orchestrator.build import build_npms +from stack_orchestrator.build import build_webapp from stack_orchestrator.deploy import deploy from stack_orchestrator import version from stack_orchestrator.deploy import deployment @@ -48,6 +49,7 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err cli.add_command(setup_repositories.command, "setup-repositories") cli.add_command(build_containers.command, "build-containers") cli.add_command(build_npms.command, "build-npms") +cli.add_command(build_webapp.command, "build-webapp") cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system cli.add_command(deploy.command, "deploy-system") cli.add_command(deployment.command, "deployment")