From 704c42c404be599c65cb954a7c1d8295091e8e00 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 23 Feb 2024 20:32:24 +0000 Subject: [PATCH 1/5] Use a catchall for single page apps. (#763) This creates a new environment variable, CERC_SINGLE_PAGE_APP, which controls whether a catchall redirection back to / is applied. If the value is not explicitly set, we try to detect if the page looks like a single-page app. Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/763 Co-authored-by: Thomas E Lackey Co-committed-by: Thomas E Lackey --- .../scripts/start-serving-app.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh index f9aa2c33..4fa1dc03 100755 --- a/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh +++ b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh @@ -3,13 +3,28 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi +CERC_LISTEN_PORT=${CERC_LISTEN_PORT:-80} CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}" CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}" +CERC_SINGLE_PAGE_APP="${CERC_SINGLE_PAGE_APP}" + +if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then + if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ] && [ -d "${CERC_WEBAPP_FILES_DIR}/static" ]; then + CERC_SINGLE_PAGE_APP=true + else + CERC_SINGLE_PAGE_APP=false + fi +fi if [ "true" == "$CERC_ENABLE_CORS" ]; then CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --cors" fi +if [ "true" == "$CERC_SINGLE_PAGE_APP" ]; then + # Create a catchall redirect back to / + CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --proxy http://localhost:${CERC_LISTEN_PORT}?" +fi + LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE} if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then if [ -f "/config/laconic-hosted-config.yml" ]; then @@ -20,8 +35,8 @@ if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then fi if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then - /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE ${CERC_WEBAPP_FILES_DIR} + /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "${CERC_WEBAPP_FILES_DIR}" fi /scripts/apply-runtime-env.sh ${CERC_WEBAPP_FILES_DIR} -http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT:-80} ${CERC_WEBAPP_FILES_DIR} +http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT} "${CERC_WEBAPP_FILES_DIR}" \ No newline at end of file From a16fc657bf2195bd7db1d05ead94d61f96874f0e Mon Sep 17 00:00:00 2001 From: zramsay Date: Sat, 24 Feb 2024 00:15:53 +0000 Subject: [PATCH 2/5] clarify uniswap urbit readme (#766) Co-authored-by: zramsay Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/766 --- .../data/stacks/uniswap-urbit-app/README.md | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md index cdbb7f6a..187d723f 100644 --- a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md +++ b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md @@ -33,23 +33,29 @@ laconic-so --stack uniswap-urbit-app deploy init --output uniswap-urbit-app-spec ### Ports -Edit `network` in spec file to map container ports to same ports in host: +Edit `uniswap-urbit-app-spec.yml` such that it looks like: ```yml -... +stack: uniswap-urbit-app +deploy-to: compose network: ports: - urbit-fake-ship: - - '8080:80' proxy-server: - '4000:4000' + urbit-fake-ship: + - '8080:80' ipfs: - - '8081:8080' - - '5001:5001' -... + - '4001' + - '8081:8080' + - 0.0.0.0:5001:5001 +volumes: + urbit_app_builds: ./data/urbit_app_builds + urbit_data: ./data/urbit_data + ipfs-import: ./data/ipfs-import + ipfs-data: ./data/ipfs-data ``` -Note: Skip the `ipfs` ports if need to use an externally running IPFS node +Note: Skip the `ipfs` ports if using an externally running IPFS node, set via `config.env`, below. ### Data volumes From a0413659f706d9e81284abd2f4c28b7fc0a706eb Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Sat, 24 Feb 2024 03:22:49 +0000 Subject: [PATCH 3/5] Check for existing tag in remote repo before building. (#764) webapps are meant to be build-once/deploy-many, but we were rebuilding them for every request. This changes that, so that we rebuild only for every unique ApplicationRecord. When we push the image, we now tag it according to its ApplicationRecord. We don't want to use that tag directly in the compose file for the deployment, however, as the deployment needs to be able to adjust to new builds w/o re-writing the file all the time. Instead, we use a per-deployment unique tag (same as before), we just update what image it references as needed. Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/764 --- stack_orchestrator/build/build_webapp.py | 15 +++++++---- stack_orchestrator/deploy/images.py | 23 +++++++++++++++++ .../webapp/deploy_webapp_from_registry.py | 25 +++++++++++++++---- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/stack_orchestrator/build/build_webapp.py b/stack_orchestrator/build/build_webapp.py index fcdd8f66..25fbc850 100644 --- a/stack_orchestrator/build/build_webapp.py +++ b/stack_orchestrator/build/build_webapp.py @@ -21,6 +21,8 @@ # TODO: display the available list of containers; allow re-build of either all or specific containers import os +import sys + from decouple import config import click from pathlib import Path @@ -40,12 +42,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t '''build the specified webapp container''' quiet = ctx.obj.quiet - verbose = ctx.obj.verbose - dry_run = ctx.obj.dry_run debug = ctx.obj.debug local_stack = ctx.obj.local_stack stack = ctx.obj.stack - continue_on_error = ctx.obj.continue_on_error # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build") @@ -73,7 +72,10 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t container_build_env, dev_root_path, ) - build_containers.process_container(build_context_1) + ok = build_containers.process_container(build_context_1) + if not ok: + print("ERROR: Build failed.", file=sys.stderr) + sys.exit(1) # Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir. container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true" @@ -94,4 +96,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t container_build_env, dev_root_path, ) - build_containers.process_container(build_context_2) + ok = build_containers.process_container(build_context_2) + if not ok: + print("ERROR: Build failed.", file=sys.stderr) + sys.exit(1) \ No newline at end of file diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py index 7ddcca33..77713d18 100644 --- a/stack_orchestrator/deploy/images.py +++ b/stack_orchestrator/deploy/images.py @@ -29,6 +29,29 @@ def _image_needs_pushed(image: str): return image.endswith(":local") +def remote_image_exists(remote_repo_url: str, local_tag: str): + docker = DockerClient() + try: + remote_tag = remote_tag_for_image(local_tag, remote_repo_url) + result = docker.manifest.inspect(remote_tag) + return True if result else False + except Exception: # noqa: E722 + return False + + +def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags): + if not additional_tags: + return + + if not remote_image_exists(remote_repo_url, local_tag): + raise Exception(f"{local_tag} does not exist in {remote_repo_url}") + + docker = DockerClient() + remote_tag = remote_tag_for_image(local_tag, remote_repo_url) + new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags] + docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags) + + def remote_tag_for_image(image: str, remote_repo_url: str): # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy major_parts = image.split("/", 2) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 15e89d9e..027a76b3 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -24,6 +24,7 @@ import uuid import click +from stack_orchestrator.deploy.images import remote_image_exists, add_tags_to_image from stack_orchestrator.deploy.webapp import deploy_webapp from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, build_container_image, push_container_image, @@ -43,6 +44,7 @@ def process_app_deployment_request( deployment_parent_dir, kube_config, image_registry, + force_rebuild=False, log_file=None ): # 1. look up application @@ -91,7 +93,9 @@ def process_app_deployment_request( deployment_record = laconic.get_record(app_deployment_crn) deployment_dir = os.path.join(deployment_parent_dir, fqdn) deployment_config_file = os.path.join(deployment_dir, "config.env") + # TODO: Is there any reason not to simplify the hash input to the app_deployment_crn? deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest() + app_image_shared_tag = f"laconic-webapp/{app.id}:local" # b. check for deployment directory (create if necessary) if not os.path.exists(deployment_dir): if deployment_record: @@ -106,11 +110,20 @@ def process_app_deployment_request( needs_k8s_deploy = False # 6. build container (if needed) if not deployment_record or deployment_record.attributes.application != app.id: - # TODO: pull from request - extra_build_args = [] - build_container_image(app, deployment_container_tag, extra_build_args, log_file) - push_container_image(deployment_dir, log_file) needs_k8s_deploy = True + # check if the image already exists + shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag) + if shared_tag_exists and not force_rebuild: + # simply add our unique tag to the existing image and we are done + print(f"Using existing app image {app_image_shared_tag} for {deployment_container_tag}", file=log_file) + add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag) + else: + extra_build_args = [] # TODO: pull from request + build_container_image(app, deployment_container_tag, extra_build_args, log_file) + push_container_image(deployment_dir, log_file) + # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag. + print(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_tag}", file=log_file) + add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag) # 7. update config (if needed) if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config: @@ -171,12 +184,13 @@ def dump_known_requests(filename, requests, status="SEEN"): @click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True) @click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="") @click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="") +@click.option("--force-rebuild", help="Rebuild even if the image already exists.", is_flag=True) @click.option("--log-dir", help="Output build/deployment logs to directory.", default=None) @click.pass_context def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, # noqa: C901 request_id, discover, state_file, only_update_state, dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run, - include_tags, exclude_tags, log_dir): + include_tags, exclude_tags, force_rebuild, log_dir): if request_id and discover: print("Cannot specify both --request-id and --discover", file=sys.stderr) sys.exit(2) @@ -306,6 +320,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ os.path.abspath(deployment_parent_dir), kube_config, image_registry, + force_rebuild, run_log_file ) status = "DEPLOYED" From 21eb9f036fff6c06ad9cc0438029b08660c045df Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Mon, 26 Feb 2024 23:31:52 +0000 Subject: [PATCH 4/5] Add support for pnpm as a webapp build tool. (#767) This adds support for auto-detecting pnpm as a build tool for webapps. Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/767 Reviewed-by: David Boreham Co-authored-by: Thomas E Lackey Co-committed-by: Thomas E Lackey --- .../cerc-nextjs-base/Dockerfile | 5 +++++ .../scripts/apply-runtime-env.sh | 8 ++++--- .../cerc-nextjs-base/scripts/build-app.sh | 4 +++- .../scripts/start-serving-app.sh | 4 +++- .../cerc-webapp-base/Dockerfile | 4 ++++ .../cerc-webapp-base/Dockerfile.webapp | 5 +++-- .../cerc-webapp-base/scripts/build-app.sh | 21 +++++++++++++++---- 7 files changed, 40 insertions(+), 11 deletions(-) diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile index 5f9548ee..d3ae8237 100644 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile @@ -26,6 +26,8 @@ RUN \ && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \ # Install semver && su ${USERNAME} -c "umask 0002 && npm install -g semver" \ + # Install pnpm + && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \ && npm cache clean --force > /dev/null 2>&1 # [Optional] Uncomment this section to install additional OS packages. @@ -35,6 +37,9 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # [Optional] Uncomment if you want to install more global node modules # RUN su node -c "npm install -g " +# We do this to get a yq binary from the published container, for the correct architecture we're building here +COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq + # Expose port for http EXPOSE 80 diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh index a662ae02..ba0f725f 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh @@ -10,10 +10,12 @@ TRG_DIR="${3:-.next-r}" CERC_BUILD_TOOL="${CERC_BUILD_TOOL}" if [ -z "$CERC_BUILD_TOOL" ]; then - if [ -f "yarn.lock" ]; then - CERC_BUILD_TOOL=npm - else + if [ -f "pnpm-lock.yaml" ]; then + CERC_BUILD_TOOL=pnpm + elif [ -f "yarn.lock" ]; then CERC_BUILD_TOOL=yarn + else + CERC_BUILD_TOOL=npm fi fi diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh index 40bcbeed..bf86265d 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh @@ -9,7 +9,9 @@ CERC_MIN_NEXTVER=13.4.2 CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}" CERC_BUILD_TOOL="${CERC_BUILD_TOOL}" if [ -z "$CERC_BUILD_TOOL" ]; then - if [ -f "yarn.lock" ]; then + if [ -f "pnpm-lock.yaml" ]; then + CERC_BUILD_TOOL=pnpm + elif [ -f "yarn.lock" ]; then CERC_BUILD_TOOL=yarn else CERC_BUILD_TOOL=npm diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh index bd254572..4b69d935 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh @@ -16,7 +16,9 @@ trap ctrl_c INT CERC_BUILD_TOOL="${CERC_BUILD_TOOL}" if [ -z "$CERC_BUILD_TOOL" ]; then - if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then + if [ -f "pnpm-lock.yaml" ]; then + CERC_BUILD_TOOL=pnpm + elif [ -f "yarn.lock" ]; then CERC_BUILD_TOOL=yarn else CERC_BUILD_TOOL=npm diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile index 9ae166b0..b8319c51 100644 --- a/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile @@ -24,6 +24,10 @@ RUN \ && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \ # Install eslint && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \ + # Install semver + && su ${USERNAME} -c "umask 0002 && npm install -g semver" \ + # Install pnpm + && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \ && npm cache clean --force > /dev/null 2>&1 # [Optional] Uncomment this section to install additional OS packages. diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp index 711eff25..9eaf46e1 100644 --- a/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp +++ b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp @@ -1,11 +1,12 @@ FROM cerc/webapp-base:local as builder ARG CERC_BUILD_TOOL +ARG CERC_BUILD_OUTPUT_DIR WORKDIR /app COPY . . -RUN rm -rf node_modules build .next* -RUN /scripts/build-app.sh /app build /data +RUN rm -rf node_modules build dist .next* +RUN /scripts/build-app.sh /app /data FROM cerc/webapp-base:local COPY --from=builder /data /data diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh index b515a142..3a63df84 100755 --- a/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh +++ b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh @@ -7,9 +7,10 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then fi CERC_BUILD_TOOL="${CERC_BUILD_TOOL}" +CERC_BUILD_OUTPUT_DIR="${CERC_BUILD_OUTPUT_DIR}" + WORK_DIR="${1:-/app}" -OUTPUT_DIR="${2:-build}" -DEST_DIR="${3:-/data}" +DEST_DIR="${2:-/data}" if [ -f "${WORK_DIR}/build-webapp.sh" ]; then echo "Building webapp with ${WORK_DIR}/build-webapp.sh ..." @@ -22,7 +23,9 @@ elif [ -f "${WORK_DIR}/package.json" ]; then cd "${WORK_DIR}" || exit 1 if [ -z "$CERC_BUILD_TOOL" ]; then - if [ -f "yarn.lock" ]; then + if [ -f "pnpm-lock.yaml" ]; then + CERC_BUILD_TOOL=pnpm + elif [ -f "yarn.lock" ]; then CERC_BUILD_TOOL=yarn else CERC_BUILD_TOOL=npm @@ -33,7 +36,17 @@ elif [ -f "${WORK_DIR}/package.json" ]; then $CERC_BUILD_TOOL build || exit 1 rm -rf "${DEST_DIR}" - mv "${WORK_DIR}/${OUTPUT_DIR}" "${DEST_DIR}" + if [ -z "${CERC_BUILD_OUTPUT_DIR}" ]; then + if [ -d "${WORK_DIR}/dist" ]; then + CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/dist" + elif [ -d "${WORK_DIR}/build" ]; then + CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/build" + else + echo "ERROR: Unable to locate build output. Set with --extra-build-args \"--build-arg CERC_BUILD_OUTPUT_DIR=path\"" 1>&2 + exit 1 + fi + fi + mv "${CERC_BUILD_OUTPUT_DIR}" "${DEST_DIR}" else echo "Copying static app ..." mv "${WORK_DIR}" "${DEST_DIR}" From da1ff609feacec6cc40317907586aae93d14e3a7 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 27 Feb 2024 15:15:08 +0000 Subject: [PATCH 5/5] fetch-images command (#768) Implementation of a command to fetch pre-built images from a remote registry, complementing the --push-images option already present on build-containers. The two subcommands used together allow a stack to be deployed without needing to built its images, provided they have been already built and pushed to the specified container image registry. This implementation simply picks the newest image with the right name and platform (matches against the platform Python is running on, so watch out for scenarios where Python is an x86 binary on M1 macs). Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/768 Co-authored-by: David Boreham Co-committed-by: David Boreham --- stack_orchestrator/build/build_containers.py | 23 +-- stack_orchestrator/build/build_util.py | 43 +++++ stack_orchestrator/build/fetch_containers.py | 180 +++++++++++++++++++ stack_orchestrator/main.py | 3 +- 4 files changed, 228 insertions(+), 21 deletions(-) create mode 100644 stack_orchestrator/build/build_util.py create mode 100644 stack_orchestrator/build/fetch_containers.py diff --git a/stack_orchestrator/build/build_containers.py b/stack_orchestrator/build/build_containers.py index 2c06afcf..71debf09 100644 --- a/stack_orchestrator/build/build_containers.py +++ b/stack_orchestrator/build/build_containers.py @@ -25,13 +25,13 @@ import sys from decouple import config import subprocess import click -import importlib.resources from pathlib import Path from stack_orchestrator.opts import opts -from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, error_exit, warn_exit +from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit from stack_orchestrator.base import get_npm_registry_url from stack_orchestrator.build.build_types import BuildContext from stack_orchestrator.build.publish import publish_image +from stack_orchestrator.build.build_util import get_containers_in_scope # TODO: find a place for this # epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)" @@ -149,24 +149,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag if not image_registry: error_exit("--image-registry must be supplied with --publish-images") - # See: https://stackoverflow.com/a/20885799/1701505 - from stack_orchestrator import data - with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file: - all_containers = container_list_file.read().splitlines() - - containers_in_scope = [] - if stack: - stack_config = get_parsed_stack_config(stack) - if "containers" not in stack_config or stack_config["containers"] is None: - warn_exit(f"stack {stack} does not define any containers") - containers_in_scope = stack_config['containers'] - else: - containers_in_scope = all_containers - - if opts.o.verbose: - print(f'Containers: {containers_in_scope}') - if stack: - print(f"Stack: {stack}") + containers_in_scope = get_containers_in_scope(stack) container_build_env = make_container_build_env(dev_root_path, container_build_dir, diff --git a/stack_orchestrator/build/build_util.py b/stack_orchestrator/build/build_util.py new file mode 100644 index 00000000..7eb89ba9 --- /dev/null +++ b/stack_orchestrator/build/build_util.py @@ -0,0 +1,43 @@ +# Copyright © 2024 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import importlib.resources + +from stack_orchestrator.opts import opts +from stack_orchestrator.util import get_parsed_stack_config, warn_exit + + +def get_containers_in_scope(stack: str): + + # See: https://stackoverflow.com/a/20885799/1701505 + from stack_orchestrator import data + with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file: + all_containers = container_list_file.read().splitlines() + + containers_in_scope = [] + if stack: + stack_config = get_parsed_stack_config(stack) + if "containers" not in stack_config or stack_config["containers"] is None: + warn_exit(f"stack {stack} does not define any containers") + containers_in_scope = stack_config['containers'] + else: + containers_in_scope = all_containers + + if opts.o.verbose: + print(f'Containers: {containers_in_scope}') + if stack: + print(f"Stack: {stack}") + + return containers_in_scope \ No newline at end of file diff --git a/stack_orchestrator/build/fetch_containers.py b/stack_orchestrator/build/fetch_containers.py new file mode 100644 index 00000000..890cb94f --- /dev/null +++ b/stack_orchestrator/build/fetch_containers.py @@ -0,0 +1,180 @@ +# Copyright © 2024 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click +from dataclasses import dataclass +import json +import platform +from python_on_whales import DockerClient +from python_on_whales.components.manifest.cli_wrapper import ManifestCLI, ManifestList +from python_on_whales.utils import run +import requests +from typing import List + +from stack_orchestrator.opts import opts +from stack_orchestrator.util import include_exclude_check, error_exit +from stack_orchestrator.build.build_util import get_containers_in_scope + +# Experimental fetch-container command + + +@dataclass +class RegistryInfo: + registry: str + registry_username: str + registry_token: str + + +# Extending this code to support the --verbose option, cnosider contributing upstream +# https://github.com/gabrieldemarmiesse/python-on-whales/blob/master/python_on_whales/components/manifest/cli_wrapper.py#L129 +class ExtendedManifestCLI(ManifestCLI): + def inspect_verbose(self, x: str) -> ManifestList: + """Returns a Docker manifest list object.""" + json_str = run(self.docker_cmd + ["manifest", "inspect", "--verbose", x]) + return json.loads(json_str) + + +def _local_tag_for(container: str): + return f"{container}:local" + + +# See: https://docker-docs.uclv.cu/registry/spec/api/ +# Emulate this: +# $ curl -u "my-username:my-token" -X GET "https:///v2/cerc-io/cerc/test-container/tags/list" +# {"name":"cerc-io/cerc/test-container","tags":["202402232130","202402232208"]} +def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List[str]: + # registry looks like: git.vdb.to/cerc-io + registry_parts = registry_info.registry.split("/") + url = f"https://{registry_parts[0]}/v2/{registry_parts[1]}/{container}/tags/list" + if opts.o.debug: + print(f"Fetching tags from: {url}") + response = requests.get(url, auth=(registry_info.registry_username, registry_info.registry_token)) + if response.status_code == 200: + tag_info = response.json() + if opts.o.debug: + print(f"container tags list: {tag_info}") + tags_array = tag_info["tags"] + return tags_array + else: + error_exit(f"failed to fetch tags from image registry, status code: {response.status_code}") + + +def _find_latest(candidate_tags: List[str]): + # Lex sort should give us the latest first + sorted_candidates = sorted(candidate_tags) + if opts.o.debug: + print(f"sorted candidates: {sorted_candidates}") + return sorted_candidates[0] + + +def _filter_for_platform(container: str, + registry_info: RegistryInfo, + tag_list: List[str]) -> List[str] : + filtered_tags = [] + this_machine = platform.machine() + # Translate between Python platform names and docker + if this_machine == "x86_64": + this_machine = "amd64" + if opts.o.debug: + print(f"Python says the architecture is: {this_machine}") + docker = DockerClient() + docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token) + for tag in tag_list: + remote_tag = f"{registry_info.registry}/{container}:{tag}" + manifest_cmd = ExtendedManifestCLI(docker.client_config) + manifest = manifest_cmd.inspect_verbose(remote_tag) + if opts.o.debug: + print(f"manifest: {manifest}") + image_architecture = manifest["Descriptor"]["platform"]["architecture"] + if opts.o.debug: + print(f"image_architecture: {image_architecture}") + if this_machine == image_architecture: + filtered_tags.append(tag) + if opts.o.debug: + print(f"Tags filtered for platform: {filtered_tags}") + return filtered_tags + + +def _get_latest_image(container: str, registry_info: RegistryInfo): + all_tags = _get_tags_for_container(container, registry_info) + tags_for_platform = _filter_for_platform(container, registry_info, all_tags) + latest_tag = _find_latest(tags_for_platform) + return f"{container}:{latest_tag}" + + +def _fetch_image(tag: str, registry_info: RegistryInfo): + docker = DockerClient() + docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token) + remote_tag = f"{registry_info.registry}/{tag}" + if opts.o.debug: + print(f"Attempting to pull this image: {remote_tag}") + docker.image.pull(remote_tag) + + +def _exists_locally(container: str): + docker = DockerClient() + return docker.image.exists(_local_tag_for(container)) + + +def _add_local_tag(remote_tag: str, registry: str, local_tag: str): + docker = DockerClient() + docker.image.tag(f"{registry}/{remote_tag}", local_tag) + + +@click.command() +@click.option('--include', help="only fetch these containers") +@click.option('--exclude', help="don\'t fetch these containers") +@click.option("--force-local-overwrite", is_flag=True, default=False, help="Overwrite a locally built image, if present") +@click.option("--image-registry", required=True, help="Specify the image registry to fetch from") +@click.option("--registry-username", required=True, help="Specify the image registry username") +@click.option("--registry-token", required=True, help="Specify the image registry access token") +@click.pass_context +def command(ctx, include, exclude, force_local_overwrite, image_registry, registry_username, registry_token): + '''EXPERIMENTAL: fetch the images for a stack from remote registry''' + + registry_info = RegistryInfo(image_registry, registry_username, registry_token) + # Generate list of target containers + stack = ctx.obj.stack + containers_in_scope = get_containers_in_scope(stack) + for container in containers_in_scope: + local_tag = _local_tag_for(container) + if include_exclude_check(container, include, exclude): + if opts.o.debug: + print(f"Processing: {container}") + # For each container, attempt to find the latest of a set of + # images with the correct name and platform in the specified registry + image_to_fetch = _get_latest_image(container, registry_info) + if opts.o.debug: + print(f"Fetching: {image_to_fetch}") + _fetch_image(image_to_fetch, registry_info) + # Now check if the target container already exists exists locally already + if (_exists_locally(container)): + if not opts.o.quiet: + print(f"Container image {container} already exists locally") + # if so, fail unless the user specified force-local-overwrite + if (force_local_overwrite): + # In that case remove the existing :local tag + if not opts.o.quiet: + print(f"Warning: overwriting local tag from this image: {container} because " + "--force-local-overwrite was specified") + else: + if not opts.o.quiet: + print(f"Skipping local tagging for this image: {container} because that would " + "overwrite an existing :local tagged image, use --force-local-overwrite to do so.") + # Tag the fetched image with the :local tag + _add_local_tag(image_to_fetch, image_registry, local_tag) + else: + if opts.o.verbose: + print(f"Excluding: {container}") diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py index a4644412..c0a49689 100644 --- a/stack_orchestrator/main.py +++ b/stack_orchestrator/main.py @@ -17,7 +17,7 @@ import click from stack_orchestrator.command_types import CommandOptions from stack_orchestrator.repos import setup_repositories -from stack_orchestrator.build import build_containers +from stack_orchestrator.build import build_containers, fetch_containers from stack_orchestrator.build import build_npms from stack_orchestrator.build import build_webapp from stack_orchestrator.deploy.webapp import (run_webapp, @@ -52,6 +52,7 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err cli.add_command(setup_repositories.command, "setup-repositories") cli.add_command(build_containers.command, "build-containers") +cli.add_command(fetch_containers.command, "fetch-containers") cli.add_command(build_npms.command, "build-npms") cli.add_command(build_webapp.command, "build-webapp") cli.add_command(run_webapp.command, "run-webapp")