From 6af45e21d16e70d3d919e9bbf03970e00fbd10e7 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Mon, 26 Feb 2024 18:45:24 -0700 Subject: [PATCH] Implement fetch-containers command --- stack_orchestrator/build/build_containers.py | 23 +--- stack_orchestrator/build/build_util.py | 43 +++++++ stack_orchestrator/build/fetch_containers.py | 125 +++++++++++++++++++ stack_orchestrator/main.py | 3 +- 4 files changed, 173 insertions(+), 21 deletions(-) create mode 100644 stack_orchestrator/build/build_util.py create mode 100644 stack_orchestrator/build/fetch_containers.py diff --git a/stack_orchestrator/build/build_containers.py b/stack_orchestrator/build/build_containers.py index 2c06afcf..71debf09 100644 --- a/stack_orchestrator/build/build_containers.py +++ b/stack_orchestrator/build/build_containers.py @@ -25,13 +25,13 @@ import sys from decouple import config import subprocess import click -import importlib.resources from pathlib import Path from stack_orchestrator.opts import opts -from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, error_exit, warn_exit +from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit from stack_orchestrator.base import get_npm_registry_url from stack_orchestrator.build.build_types import BuildContext from stack_orchestrator.build.publish import publish_image +from stack_orchestrator.build.build_util import get_containers_in_scope # TODO: find a place for this # epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)" @@ -149,24 +149,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag if not image_registry: error_exit("--image-registry must be supplied with --publish-images") - # See: https://stackoverflow.com/a/20885799/1701505 - from stack_orchestrator import data - with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file: - all_containers = container_list_file.read().splitlines() - - containers_in_scope = [] - if stack: - stack_config = get_parsed_stack_config(stack) - if "containers" not in stack_config or stack_config["containers"] is None: - warn_exit(f"stack {stack} does not define any containers") - containers_in_scope = stack_config['containers'] - else: - containers_in_scope = all_containers - - if opts.o.verbose: - print(f'Containers: {containers_in_scope}') - if stack: - print(f"Stack: {stack}") + containers_in_scope = get_containers_in_scope(stack) container_build_env = make_container_build_env(dev_root_path, container_build_dir, diff --git a/stack_orchestrator/build/build_util.py b/stack_orchestrator/build/build_util.py new file mode 100644 index 00000000..7eb89ba9 --- /dev/null +++ b/stack_orchestrator/build/build_util.py @@ -0,0 +1,43 @@ +# Copyright © 2024 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import importlib.resources + +from stack_orchestrator.opts import opts +from stack_orchestrator.util import get_parsed_stack_config, warn_exit + + +def get_containers_in_scope(stack: str): + + # See: https://stackoverflow.com/a/20885799/1701505 + from stack_orchestrator import data + with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file: + all_containers = container_list_file.read().splitlines() + + containers_in_scope = [] + if stack: + stack_config = get_parsed_stack_config(stack) + if "containers" not in stack_config or stack_config["containers"] is None: + warn_exit(f"stack {stack} does not define any containers") + containers_in_scope = stack_config['containers'] + else: + containers_in_scope = all_containers + + if opts.o.verbose: + print(f'Containers: {containers_in_scope}') + if stack: + print(f"Stack: {stack}") + + return containers_in_scope \ No newline at end of file diff --git a/stack_orchestrator/build/fetch_containers.py b/stack_orchestrator/build/fetch_containers.py new file mode 100644 index 00000000..66db1848 --- /dev/null +++ b/stack_orchestrator/build/fetch_containers.py @@ -0,0 +1,125 @@ +# Copyright © 2024 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click +from python_on_whales import DockerClient +import requests +from typing import List + +from stack_orchestrator.opts import opts +from stack_orchestrator.util import include_exclude_check, error_exit +from stack_orchestrator.build.build_util import get_containers_in_scope + +# Experimental fetch-container command + + +def _local_tag_for(container: str): + return f"{container}:local" + +# Emulate this: +# $ curl -u "my-username:my-token" -X GET "https:///v2/cerc-io/cerc/test-container/tags/list" +# {"name":"cerc-io/cerc/test-container","tags":["202402232130","202402232208"]} +def _get_tags_for_container(container: str, registry: str) -> List[str]: + # registry looks like: git.vdb.to/cerc-io + registry_parts = registry.split("/") + url = f"https://{registry_parts[0]}/v2/{registry_parts[1]}/{container}/tags/list" + if opts.o.debug: + print(f"Fetching tags from: {url}") + response = requests.get(url, auth=("dboreham", "fb44b8dd36638bcf4abf478f7b1c3e18ed578f10")) + if response.status_code == 200: + tag_info = response.json() + if opts.o.debug: + print(f"Got this: {tag_info}") + tags_array = tag_info["tags"] + return tags_array + else: + error_exit(f"failed to fetch tags from image registry, status code: {response.status_code}") + + +def _find_latest(candidate_tags: List[str]): + # HACK: return the first one for now + return candidate_tags[0] + + +def _get_latest_image(container: str, registry: str): + all_tags = _get_tags_for_container(container, registry) + latest_tag = _find_latest(all_tags) + return f"{container}:{latest_tag}" + + +def _fetch_image(tag: str, registry: str): + docker = DockerClient() + remote_tag = f"{registry}/{tag}" + if opts.o.debug: + print(f"Attempting to pull this image: {remote_tag}") + docker.image.pull(remote_tag) + + +def _exists_locally(container: str): + docker = DockerClient() + return docker.image.exists(_local_tag_for(container)) + + +def _remove_local_tag(container: str): + # TODO Do we need this? + pass + + +def _add_local_tag(remote_tag: str, registry: str, local_tag: str): + docker = DockerClient() + docker.image.tag(f"{registry}/{remote_tag}", local_tag) + + +@click.command() +@click.option('--include', help="only fetch these containers") +@click.option('--exclude', help="don\'t fetch these containers") +@click.option("--force-local-overwrite", is_flag=True, default=False, help="Overwrite a locally built image, if present") +@click.option("--image-registry", required=True, help="Specify the image registry to fetch from") +@click.pass_context +def command(ctx, include, exclude, force_local_overwrite, image_registry): + '''EXPERIMENTAL: fetch the images for a stack from remote registry''' + + # Generate list of target containers + stack = ctx.obj.stack + containers_in_scope = get_containers_in_scope(stack) + for container in containers_in_scope: + local_tag = _local_tag_for(container) + if include_exclude_check(container, include, exclude): + if opts.o.debug: + print(f"Processing: {container}") + # For each container, attempt to find the latest of a set of + # images with the correct name and platform in the specified registry + image_to_fetch = _get_latest_image(container, image_registry) + if opts.o.debug: + print(f"Fetching: {image_to_fetch}") + _fetch_image(image_to_fetch, image_registry) + # Now check if the target container already exists exists locally already + if (_exists_locally(container)): + if not opts.o.quiet: + print(f"Container image {container} already exists locally") + # if so, fail unless the user specified force-local-overwrite + if (force_local_overwrite): + # In that case remove the existing :local tag + if not opts.o.quiet: + print(f"Warning: removing local tag from this image: {container} because --force-local-overwrite was specified") + _remove_local_tag(local_tag) + else: + if not opts.o.quiet: + print(f"Skipping download for this image: {container}") + # Tag the fetched image with the :local tag + _add_local_tag(image_to_fetch, image_registry, local_tag) + else: + if opts.o.verbose: + print(f"Excluding: {container}") diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py index a4644412..c0a49689 100644 --- a/stack_orchestrator/main.py +++ b/stack_orchestrator/main.py @@ -17,7 +17,7 @@ import click from stack_orchestrator.command_types import CommandOptions from stack_orchestrator.repos import setup_repositories -from stack_orchestrator.build import build_containers +from stack_orchestrator.build import build_containers, fetch_containers from stack_orchestrator.build import build_npms from stack_orchestrator.build import build_webapp from stack_orchestrator.deploy.webapp import (run_webapp, @@ -52,6 +52,7 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err cli.add_command(setup_repositories.command, "setup-repositories") cli.add_command(build_containers.command, "build-containers") +cli.add_command(fetch_containers.command, "fetch-containers") cli.add_command(build_npms.command, "build-npms") cli.add_command(build_webapp.command, "build-webapp") cli.add_command(run_webapp.command, "run-webapp")