From da1ff609feacec6cc40317907586aae93d14e3a7 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 27 Feb 2024 15:15:08 +0000 Subject: [PATCH] fetch-images command (#768) Implementation of a command to fetch pre-built images from a remote registry, complementing the --push-images option already present on build-containers. The two subcommands used together allow a stack to be deployed without needing to built its images, provided they have been already built and pushed to the specified container image registry. This implementation simply picks the newest image with the right name and platform (matches against the platform Python is running on, so watch out for scenarios where Python is an x86 binary on M1 macs). Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/768 Co-authored-by: David Boreham Co-committed-by: David Boreham --- stack_orchestrator/build/build_containers.py | 23 +-- stack_orchestrator/build/build_util.py | 43 +++++ stack_orchestrator/build/fetch_containers.py | 180 +++++++++++++++++++ stack_orchestrator/main.py | 3 +- 4 files changed, 228 insertions(+), 21 deletions(-) create mode 100644 stack_orchestrator/build/build_util.py create mode 100644 stack_orchestrator/build/fetch_containers.py diff --git a/stack_orchestrator/build/build_containers.py b/stack_orchestrator/build/build_containers.py index 2c06afcf..71debf09 100644 --- a/stack_orchestrator/build/build_containers.py +++ b/stack_orchestrator/build/build_containers.py @@ -25,13 +25,13 @@ import sys from decouple import config import subprocess import click -import importlib.resources from pathlib import Path from stack_orchestrator.opts import opts -from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, error_exit, warn_exit +from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit from stack_orchestrator.base import get_npm_registry_url from stack_orchestrator.build.build_types import BuildContext from stack_orchestrator.build.publish import publish_image +from stack_orchestrator.build.build_util import get_containers_in_scope # TODO: find a place for this # epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)" @@ -149,24 +149,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag if not image_registry: error_exit("--image-registry must be supplied with --publish-images") - # See: https://stackoverflow.com/a/20885799/1701505 - from stack_orchestrator import data - with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file: - all_containers = container_list_file.read().splitlines() - - containers_in_scope = [] - if stack: - stack_config = get_parsed_stack_config(stack) - if "containers" not in stack_config or stack_config["containers"] is None: - warn_exit(f"stack {stack} does not define any containers") - containers_in_scope = stack_config['containers'] - else: - containers_in_scope = all_containers - - if opts.o.verbose: - print(f'Containers: {containers_in_scope}') - if stack: - print(f"Stack: {stack}") + containers_in_scope = get_containers_in_scope(stack) container_build_env = make_container_build_env(dev_root_path, container_build_dir, diff --git a/stack_orchestrator/build/build_util.py b/stack_orchestrator/build/build_util.py new file mode 100644 index 00000000..7eb89ba9 --- /dev/null +++ b/stack_orchestrator/build/build_util.py @@ -0,0 +1,43 @@ +# Copyright © 2024 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import importlib.resources + +from stack_orchestrator.opts import opts +from stack_orchestrator.util import get_parsed_stack_config, warn_exit + + +def get_containers_in_scope(stack: str): + + # See: https://stackoverflow.com/a/20885799/1701505 + from stack_orchestrator import data + with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file: + all_containers = container_list_file.read().splitlines() + + containers_in_scope = [] + if stack: + stack_config = get_parsed_stack_config(stack) + if "containers" not in stack_config or stack_config["containers"] is None: + warn_exit(f"stack {stack} does not define any containers") + containers_in_scope = stack_config['containers'] + else: + containers_in_scope = all_containers + + if opts.o.verbose: + print(f'Containers: {containers_in_scope}') + if stack: + print(f"Stack: {stack}") + + return containers_in_scope \ No newline at end of file diff --git a/stack_orchestrator/build/fetch_containers.py b/stack_orchestrator/build/fetch_containers.py new file mode 100644 index 00000000..890cb94f --- /dev/null +++ b/stack_orchestrator/build/fetch_containers.py @@ -0,0 +1,180 @@ +# Copyright © 2024 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click +from dataclasses import dataclass +import json +import platform +from python_on_whales import DockerClient +from python_on_whales.components.manifest.cli_wrapper import ManifestCLI, ManifestList +from python_on_whales.utils import run +import requests +from typing import List + +from stack_orchestrator.opts import opts +from stack_orchestrator.util import include_exclude_check, error_exit +from stack_orchestrator.build.build_util import get_containers_in_scope + +# Experimental fetch-container command + + +@dataclass +class RegistryInfo: + registry: str + registry_username: str + registry_token: str + + +# Extending this code to support the --verbose option, cnosider contributing upstream +# https://github.com/gabrieldemarmiesse/python-on-whales/blob/master/python_on_whales/components/manifest/cli_wrapper.py#L129 +class ExtendedManifestCLI(ManifestCLI): + def inspect_verbose(self, x: str) -> ManifestList: + """Returns a Docker manifest list object.""" + json_str = run(self.docker_cmd + ["manifest", "inspect", "--verbose", x]) + return json.loads(json_str) + + +def _local_tag_for(container: str): + return f"{container}:local" + + +# See: https://docker-docs.uclv.cu/registry/spec/api/ +# Emulate this: +# $ curl -u "my-username:my-token" -X GET "https:///v2/cerc-io/cerc/test-container/tags/list" +# {"name":"cerc-io/cerc/test-container","tags":["202402232130","202402232208"]} +def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List[str]: + # registry looks like: git.vdb.to/cerc-io + registry_parts = registry_info.registry.split("/") + url = f"https://{registry_parts[0]}/v2/{registry_parts[1]}/{container}/tags/list" + if opts.o.debug: + print(f"Fetching tags from: {url}") + response = requests.get(url, auth=(registry_info.registry_username, registry_info.registry_token)) + if response.status_code == 200: + tag_info = response.json() + if opts.o.debug: + print(f"container tags list: {tag_info}") + tags_array = tag_info["tags"] + return tags_array + else: + error_exit(f"failed to fetch tags from image registry, status code: {response.status_code}") + + +def _find_latest(candidate_tags: List[str]): + # Lex sort should give us the latest first + sorted_candidates = sorted(candidate_tags) + if opts.o.debug: + print(f"sorted candidates: {sorted_candidates}") + return sorted_candidates[0] + + +def _filter_for_platform(container: str, + registry_info: RegistryInfo, + tag_list: List[str]) -> List[str] : + filtered_tags = [] + this_machine = platform.machine() + # Translate between Python platform names and docker + if this_machine == "x86_64": + this_machine = "amd64" + if opts.o.debug: + print(f"Python says the architecture is: {this_machine}") + docker = DockerClient() + docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token) + for tag in tag_list: + remote_tag = f"{registry_info.registry}/{container}:{tag}" + manifest_cmd = ExtendedManifestCLI(docker.client_config) + manifest = manifest_cmd.inspect_verbose(remote_tag) + if opts.o.debug: + print(f"manifest: {manifest}") + image_architecture = manifest["Descriptor"]["platform"]["architecture"] + if opts.o.debug: + print(f"image_architecture: {image_architecture}") + if this_machine == image_architecture: + filtered_tags.append(tag) + if opts.o.debug: + print(f"Tags filtered for platform: {filtered_tags}") + return filtered_tags + + +def _get_latest_image(container: str, registry_info: RegistryInfo): + all_tags = _get_tags_for_container(container, registry_info) + tags_for_platform = _filter_for_platform(container, registry_info, all_tags) + latest_tag = _find_latest(tags_for_platform) + return f"{container}:{latest_tag}" + + +def _fetch_image(tag: str, registry_info: RegistryInfo): + docker = DockerClient() + docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token) + remote_tag = f"{registry_info.registry}/{tag}" + if opts.o.debug: + print(f"Attempting to pull this image: {remote_tag}") + docker.image.pull(remote_tag) + + +def _exists_locally(container: str): + docker = DockerClient() + return docker.image.exists(_local_tag_for(container)) + + +def _add_local_tag(remote_tag: str, registry: str, local_tag: str): + docker = DockerClient() + docker.image.tag(f"{registry}/{remote_tag}", local_tag) + + +@click.command() +@click.option('--include', help="only fetch these containers") +@click.option('--exclude', help="don\'t fetch these containers") +@click.option("--force-local-overwrite", is_flag=True, default=False, help="Overwrite a locally built image, if present") +@click.option("--image-registry", required=True, help="Specify the image registry to fetch from") +@click.option("--registry-username", required=True, help="Specify the image registry username") +@click.option("--registry-token", required=True, help="Specify the image registry access token") +@click.pass_context +def command(ctx, include, exclude, force_local_overwrite, image_registry, registry_username, registry_token): + '''EXPERIMENTAL: fetch the images for a stack from remote registry''' + + registry_info = RegistryInfo(image_registry, registry_username, registry_token) + # Generate list of target containers + stack = ctx.obj.stack + containers_in_scope = get_containers_in_scope(stack) + for container in containers_in_scope: + local_tag = _local_tag_for(container) + if include_exclude_check(container, include, exclude): + if opts.o.debug: + print(f"Processing: {container}") + # For each container, attempt to find the latest of a set of + # images with the correct name and platform in the specified registry + image_to_fetch = _get_latest_image(container, registry_info) + if opts.o.debug: + print(f"Fetching: {image_to_fetch}") + _fetch_image(image_to_fetch, registry_info) + # Now check if the target container already exists exists locally already + if (_exists_locally(container)): + if not opts.o.quiet: + print(f"Container image {container} already exists locally") + # if so, fail unless the user specified force-local-overwrite + if (force_local_overwrite): + # In that case remove the existing :local tag + if not opts.o.quiet: + print(f"Warning: overwriting local tag from this image: {container} because " + "--force-local-overwrite was specified") + else: + if not opts.o.quiet: + print(f"Skipping local tagging for this image: {container} because that would " + "overwrite an existing :local tagged image, use --force-local-overwrite to do so.") + # Tag the fetched image with the :local tag + _add_local_tag(image_to_fetch, image_registry, local_tag) + else: + if opts.o.verbose: + print(f"Excluding: {container}") diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py index a4644412..c0a49689 100644 --- a/stack_orchestrator/main.py +++ b/stack_orchestrator/main.py @@ -17,7 +17,7 @@ import click from stack_orchestrator.command_types import CommandOptions from stack_orchestrator.repos import setup_repositories -from stack_orchestrator.build import build_containers +from stack_orchestrator.build import build_containers, fetch_containers from stack_orchestrator.build import build_npms from stack_orchestrator.build import build_webapp from stack_orchestrator.deploy.webapp import (run_webapp, @@ -52,6 +52,7 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err cli.add_command(setup_repositories.command, "setup-repositories") cli.add_command(build_containers.command, "build-containers") +cli.add_command(fetch_containers.command, "fetch-containers") cli.add_command(build_npms.command, "build-npms") cli.add_command(build_webapp.command, "build-webapp") cli.add_command(run_webapp.command, "run-webapp")