From 6629017d6a987f77fede8580843947b6715976dc Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 2 Feb 2024 18:04:06 -0600 Subject: [PATCH] Support other webapp types (react, static). (#721) * Support other webapp types (react, static). --- stack_orchestrator/build/build_webapp.py | 11 +++--- .../cerc-nextjs-base/Dockerfile | 2 +- .../scripts/start-serving-app.sh | 2 +- .../cerc-webapp-base/Dockerfile | 8 ++--- .../cerc-webapp-base/Dockerfile.webapp | 11 ++++++ .../container-build/cerc-webapp-base/build.sh | 24 +++++++++++-- .../scripts/apply-runtime-env.sh | 33 +++++++++++++++++ .../{ => scripts}/apply-webapp-config.sh | 0 .../cerc-webapp-base/scripts/build-app.sh | 36 +++++++++++++++++++ .../scripts/start-serving-app.sh | 15 ++++++++ .../cerc-webapp-base/start-serving-app.sh | 9 ----- .../deploy/webapp/run_webapp.py | 2 +- stack_orchestrator/deploy/webapp/util.py | 26 +++++++++++++- tests/webapp-test/run-webapp-test.sh | 4 +-- 14 files changed, 157 insertions(+), 26 deletions(-) create mode 100644 stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp create mode 100755 stack_orchestrator/data/container-build/cerc-webapp-base/scripts/apply-runtime-env.sh rename stack_orchestrator/data/container-build/cerc-webapp-base/{ => scripts}/apply-webapp-config.sh (100%) create mode 100755 stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh create mode 100755 stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh delete mode 100755 stack_orchestrator/data/container-build/cerc-webapp-base/start-serving-app.sh diff --git a/stack_orchestrator/build/build_webapp.py b/stack_orchestrator/build/build_webapp.py index 287347eb..a9124590 100644 --- a/stack_orchestrator/build/build_webapp.py +++ b/stack_orchestrator/build/build_webapp.py @@ -25,10 +25,11 @@ from decouple import config import click from pathlib import Path from stack_orchestrator.build import build_containers +from stack_orchestrator.deploy.webapp.util import determine_base_container @click.command() -@click.option('--base-container', default="cerc/nextjs-base") +@click.option('--base-container') @click.option('--source-repo', help="directory containing the webapp to build", required=True) @click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild") @click.option("--extra-build-args", help="Supply extra arguments to build") @@ -57,6 +58,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t if not quiet: print(f'Dev Root is: {dev_root_path}') + if not base_container: + base_container = determine_base_container(source_repo) + # First build the base container. container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug, force_rebuild, extra_build_args) @@ -64,13 +68,12 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet, verbose, dry_run, continue_on_error) - # Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir. container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true" container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo) container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir, - base_container.replace("/", "-"), - "Dockerfile.webapp") + base_container.replace("/", "-"), + "Dockerfile.webapp") if not tag: webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1] container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local" diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile index c3c12cc6..5f9548ee 100644 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # RUN su node -c "npm install -g " # Expose port for http -EXPOSE 3000 +EXPOSE 80 COPY /scripts /scripts diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh index 3692f2f6..bd254572 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh @@ -58,4 +58,4 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then fi fi -$CERC_BUILD_TOOL start . -p ${CERC_LISTEN_PORT:-3000} +$CERC_BUILD_TOOL start . -- -p ${CERC_LISTEN_PORT:-80} diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile index 275a5c3c..331e04a1 100644 --- a/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile @@ -1,6 +1,6 @@ # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster -ARG VARIANT=18-bullseye +ARG VARIANT=20-bullseye FROM node:${VARIANT} ARG USERNAME=node @@ -28,7 +28,7 @@ RUN \ # [Optional] Uncomment this section to install additional OS packages. RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends jq + && apt-get -y install --no-install-recommends jq gettext-base # [Optional] Uncomment if you want to install an additional version of node using nvm # ARG EXTRA_NODE_VERSION=10 @@ -37,9 +37,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # We do this to get a yq binary from the published container, for the correct architecture we're building here COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq -RUN mkdir -p /scripts -COPY ./apply-webapp-config.sh /scripts -COPY ./start-serving-app.sh /scripts +COPY scripts /scripts # [Optional] Uncomment if you want to install more global node modules # RUN su node -c "npm install -g " diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp new file mode 100644 index 00000000..711eff25 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp @@ -0,0 +1,11 @@ +FROM cerc/webapp-base:local as builder + +ARG CERC_BUILD_TOOL + +WORKDIR /app +COPY . . +RUN rm -rf node_modules build .next* +RUN /scripts/build-app.sh /app build /data + +FROM cerc/webapp-base:local +COPY --from=builder /data /data diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/build.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/build.sh index 51712dc4..12b25e5e 100755 --- a/stack_orchestrator/data/container-build/cerc-webapp-base/build.sh +++ b/stack_orchestrator/data/container-build/cerc-webapp-base/build.sh @@ -1,9 +1,29 @@ #!/usr/bin/env bash -# Build cerc/laconic-registry-cli +# Build cerc/webapp-base source ${CERC_CONTAINER_BASE_DIR}/build-base.sh # See: https://stackoverflow.com/a/246128/1701505 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -docker build -t cerc/webapp-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile ${SCRIPT_DIR} +CERC_CONTAINER_BUILD_WORK_DIR=${CERC_CONTAINER_BUILD_WORK_DIR:-$SCRIPT_DIR} +CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/Dockerfile} +CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/webapp-base:local} + +docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR + +if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/webapp-base:local" ]; then + cat < $TMP_ENV + set -a + source .env + source $TMP_ENV + set +a + rm -f $TMP_ENV +fi + +for f in $(find . -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do + for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},();]' '\n' | egrep -o -e '^"CERC_RUNTIME_ENV_[^\"]+"' -e '^"LACONIC_HOSTED_CONFIG_[^\"]+"'); do + orig_name=$(echo -n "${e}" | sed 's/"//g') + cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g') + cur_val=$(echo -n "\$${cur_name}" | envsubst) + if [ "$CERC_RETAIN_ENV_QUOTES" != "true" ]; then + cur_val=$(sed "s/^[\"']//" <<< "$cur_val" | sed "s/[\"']//") + fi + esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val") + echo "$f: $cur_name=$cur_val" + sed -i "s/$orig_name/$esc_val/g" $f + done +done diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/apply-webapp-config.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/apply-webapp-config.sh similarity index 100% rename from stack_orchestrator/data/container-build/cerc-webapp-base/apply-webapp-config.sh rename to stack_orchestrator/data/container-build/cerc-webapp-base/scripts/apply-webapp-config.sh diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh new file mode 100755 index 00000000..c102a054 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +CERC_BUILD_TOOL="${CERC_BUILD_TOOL}" +WORK_DIR="${1:-/app}" +OUTPUT_DIR="${2:-build}" +DEST_DIR="${3:-/data}" + +if [ -f "${WORK_DIR}/package.json" ]; then + echo "Building node-based webapp ..." + cd "${WORK_DIR}" || exit 1 + + if [ -z "$CERC_BUILD_TOOL" ]; then + if [ -f "yarn.lock" ]; then + CERC_BUILD_TOOL=yarn + else + CERC_BUILD_TOOL=npm + fi + fi + + $CERC_BUILD_TOOL install || exit 1 + $CERC_BUILD_TOOL build || exit 1 + + rm -rf "${DEST_DIR}" + mv "${WORK_DIR}/${OUTPUT_DIR}" "${DEST_DIR}" +else + echo "Copying static app ..." + mv "${WORK_DIR}" "${DEST_DIR}" +fi + +exit 0 diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh new file mode 100755 index 00000000..365d05fb --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}" +CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}" + +if [ "true" == "$CERC_ENABLE_CORS" ]; then + CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --cors" +fi + +/scripts/apply-webapp-config.sh /config/config.yml ${CERC_WEBAPP_FILES_DIR} +/scripts/apply-runtime-env.sh ${CERC_WEBAPP_FILES_DIR} +http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT:-80} ${CERC_WEBAPP_FILES_DIR} diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/start-serving-app.sh deleted file mode 100755 index 69fa6c22..00000000 --- a/stack_orchestrator/data/container-build/cerc-webapp-base/start-serving-app.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -if [ -n "$CERC_SCRIPT_DEBUG" ]; then - set -x -fi - -CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}" - -/scripts/apply-webapp-config.sh /config/config.yml ${CERC_WEBAPP_FILES_DIR} -http-server -p 80 ${CERC_WEBAPP_FILES_DIR} diff --git a/stack_orchestrator/deploy/webapp/run_webapp.py b/stack_orchestrator/deploy/webapp/run_webapp.py index 4dbf234a..f780c6f8 100644 --- a/stack_orchestrator/deploy/webapp/run_webapp.py +++ b/stack_orchestrator/deploy/webapp/run_webapp.py @@ -27,7 +27,7 @@ from dotenv import dotenv_values from stack_orchestrator import constants from stack_orchestrator.deploy.deployer_factory import getDeployer -WEBAPP_PORT = 3000 +WEBAPP_PORT = 80 @click.command() diff --git a/stack_orchestrator/deploy/webapp/util.py b/stack_orchestrator/deploy/webapp/util.py index ef98117b..50485110 100644 --- a/stack_orchestrator/deploy/webapp/util.py +++ b/stack_orchestrator/deploy/webapp/util.py @@ -195,6 +195,23 @@ def file_hash(filename): return hashlib.sha1(open(filename).read().encode()).hexdigest() +def determine_base_container(clone_dir, app_type="webapp"): + if not app_type or not app_type.startswith("webapp"): + raise Exception(f"Unsupported app_type {app_type}") + + base_container = "cerc/webapp-base" + if app_type == "webapp/next": + base_container = "cerc/nextjs-base" + elif app_type == "webapp": + pkg_json_path = os.path.join(clone_dir, "package.json") + if os.path.exists(pkg_json_path): + pkg_json = json.load(open(pkg_json_path)) + if "next" in pkg_json.get("dependencies", {}): + base_container = "cerc/nextjs-base" + + return base_container + + def build_container_image(app_record, tag, extra_build_args=[]): tmpdir = tempfile.mkdtemp() @@ -216,8 +233,15 @@ def build_container_image(app_record, tag, extra_build_args=[]): result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir]) result.check_returncode() + base_container = determine_base_container(clone_dir, app_record.attributes.app_type) + print("Building webapp ...") - build_command = [sys.argv[0], "build-webapp", "--source-repo", clone_dir, "--tag", tag] + build_command = [ + sys.argv[0], "build-webapp", + "--source-repo", clone_dir, + "--tag", tag, + "--base-container", base_container + ] if extra_build_args: build_command.append("--extra-build-args") build_command.append(" ".join(extra_build_args)) diff --git a/tests/webapp-test/run-webapp-test.sh b/tests/webapp-test/run-webapp-test.sh index 5db382f8..8cae4828 100755 --- a/tests/webapp-test/run-webapp-test.sh +++ b/tests/webapp-test/run-webapp-test.sh @@ -30,14 +30,14 @@ CHECK="SPECIAL_01234567890_TEST_STRING" set +e -CONTAINER_ID=$(docker run -p 3000:3000 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local) +CONTAINER_ID=$(docker run -p 3000:80 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local) sleep 3 wget -t 7 -O test.before -m http://localhost:3000 docker logs $CONTAINER_ID docker remove -f $CONTAINER_ID -CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local) +CONTAINER_ID=$(docker run -p 3000:80 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local) sleep 3 wget -t 7 -O test.after -m http://localhost:3000