fetch-images command (#768)
All checks were successful
Lint Checks / Run linter (push) Successful in 35s
Publish / Build and publish (push) Successful in 1m33s
Webapp Test / Run webapp test suite (push) Successful in 3m44s
Smoke Test / Run basic test suite (push) Successful in 4m9s
Deploy Test / Run deploy test suite (push) Successful in 7m3s
All checks were successful
Lint Checks / Run linter (push) Successful in 35s
Publish / Build and publish (push) Successful in 1m33s
Webapp Test / Run webapp test suite (push) Successful in 3m44s
Smoke Test / Run basic test suite (push) Successful in 4m9s
Deploy Test / Run deploy test suite (push) Successful in 7m3s
Implementation of a command to fetch pre-built images from a remote registry, complementing the --push-images option already present on build-containers. The two subcommands used together allow a stack to be deployed without needing to built its images, provided they have been already built and pushed to the specified container image registry. This implementation simply picks the newest image with the right name and platform (matches against the platform Python is running on, so watch out for scenarios where Python is an x86 binary on M1 macs). Reviewed-on: #768 Co-authored-by: David Boreham <david@bozemanpass.com> Co-committed-by: David Boreham <david@bozemanpass.com>
This commit is contained in:
parent
21eb9f036f
commit
da1ff609fe
@ -25,13 +25,13 @@ import sys
|
||||
from decouple import config
|
||||
import subprocess
|
||||
import click
|
||||
import importlib.resources
|
||||
from pathlib import Path
|
||||
from stack_orchestrator.opts import opts
|
||||
from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, error_exit, warn_exit
|
||||
from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
|
||||
from stack_orchestrator.base import get_npm_registry_url
|
||||
from stack_orchestrator.build.build_types import BuildContext
|
||||
from stack_orchestrator.build.publish import publish_image
|
||||
from stack_orchestrator.build.build_util import get_containers_in_scope
|
||||
|
||||
# TODO: find a place for this
|
||||
# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
|
||||
@ -149,24 +149,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag
|
||||
if not image_registry:
|
||||
error_exit("--image-registry must be supplied with --publish-images")
|
||||
|
||||
# See: https://stackoverflow.com/a/20885799/1701505
|
||||
from stack_orchestrator import data
|
||||
with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
|
||||
all_containers = container_list_file.read().splitlines()
|
||||
|
||||
containers_in_scope = []
|
||||
if stack:
|
||||
stack_config = get_parsed_stack_config(stack)
|
||||
if "containers" not in stack_config or stack_config["containers"] is None:
|
||||
warn_exit(f"stack {stack} does not define any containers")
|
||||
containers_in_scope = stack_config['containers']
|
||||
else:
|
||||
containers_in_scope = all_containers
|
||||
|
||||
if opts.o.verbose:
|
||||
print(f'Containers: {containers_in_scope}')
|
||||
if stack:
|
||||
print(f"Stack: {stack}")
|
||||
containers_in_scope = get_containers_in_scope(stack)
|
||||
|
||||
container_build_env = make_container_build_env(dev_root_path,
|
||||
container_build_dir,
|
||||
|
43
stack_orchestrator/build/build_util.py
Normal file
43
stack_orchestrator/build/build_util.py
Normal file
@ -0,0 +1,43 @@
|
||||
# Copyright © 2024 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
import importlib.resources
|
||||
|
||||
from stack_orchestrator.opts import opts
|
||||
from stack_orchestrator.util import get_parsed_stack_config, warn_exit
|
||||
|
||||
|
||||
def get_containers_in_scope(stack: str):
|
||||
|
||||
# See: https://stackoverflow.com/a/20885799/1701505
|
||||
from stack_orchestrator import data
|
||||
with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
|
||||
all_containers = container_list_file.read().splitlines()
|
||||
|
||||
containers_in_scope = []
|
||||
if stack:
|
||||
stack_config = get_parsed_stack_config(stack)
|
||||
if "containers" not in stack_config or stack_config["containers"] is None:
|
||||
warn_exit(f"stack {stack} does not define any containers")
|
||||
containers_in_scope = stack_config['containers']
|
||||
else:
|
||||
containers_in_scope = all_containers
|
||||
|
||||
if opts.o.verbose:
|
||||
print(f'Containers: {containers_in_scope}')
|
||||
if stack:
|
||||
print(f"Stack: {stack}")
|
||||
|
||||
return containers_in_scope
|
180
stack_orchestrator/build/fetch_containers.py
Normal file
180
stack_orchestrator/build/fetch_containers.py
Normal file
@ -0,0 +1,180 @@
|
||||
# Copyright © 2024 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
import click
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
import platform
|
||||
from python_on_whales import DockerClient
|
||||
from python_on_whales.components.manifest.cli_wrapper import ManifestCLI, ManifestList
|
||||
from python_on_whales.utils import run
|
||||
import requests
|
||||
from typing import List
|
||||
|
||||
from stack_orchestrator.opts import opts
|
||||
from stack_orchestrator.util import include_exclude_check, error_exit
|
||||
from stack_orchestrator.build.build_util import get_containers_in_scope
|
||||
|
||||
# Experimental fetch-container command
|
||||
|
||||
|
||||
@dataclass
|
||||
class RegistryInfo:
|
||||
registry: str
|
||||
registry_username: str
|
||||
registry_token: str
|
||||
|
||||
|
||||
# Extending this code to support the --verbose option, cnosider contributing upstream
|
||||
# https://github.com/gabrieldemarmiesse/python-on-whales/blob/master/python_on_whales/components/manifest/cli_wrapper.py#L129
|
||||
class ExtendedManifestCLI(ManifestCLI):
|
||||
def inspect_verbose(self, x: str) -> ManifestList:
|
||||
"""Returns a Docker manifest list object."""
|
||||
json_str = run(self.docker_cmd + ["manifest", "inspect", "--verbose", x])
|
||||
return json.loads(json_str)
|
||||
|
||||
|
||||
def _local_tag_for(container: str):
|
||||
return f"{container}:local"
|
||||
|
||||
|
||||
# See: https://docker-docs.uclv.cu/registry/spec/api/
|
||||
# Emulate this:
|
||||
# $ curl -u "my-username:my-token" -X GET "https://<container-registry-hostname>/v2/cerc-io/cerc/test-container/tags/list"
|
||||
# {"name":"cerc-io/cerc/test-container","tags":["202402232130","202402232208"]}
|
||||
def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List[str]:
|
||||
# registry looks like: git.vdb.to/cerc-io
|
||||
registry_parts = registry_info.registry.split("/")
|
||||
url = f"https://{registry_parts[0]}/v2/{registry_parts[1]}/{container}/tags/list"
|
||||
if opts.o.debug:
|
||||
print(f"Fetching tags from: {url}")
|
||||
response = requests.get(url, auth=(registry_info.registry_username, registry_info.registry_token))
|
||||
if response.status_code == 200:
|
||||
tag_info = response.json()
|
||||
if opts.o.debug:
|
||||
print(f"container tags list: {tag_info}")
|
||||
tags_array = tag_info["tags"]
|
||||
return tags_array
|
||||
else:
|
||||
error_exit(f"failed to fetch tags from image registry, status code: {response.status_code}")
|
||||
|
||||
|
||||
def _find_latest(candidate_tags: List[str]):
|
||||
# Lex sort should give us the latest first
|
||||
sorted_candidates = sorted(candidate_tags)
|
||||
if opts.o.debug:
|
||||
print(f"sorted candidates: {sorted_candidates}")
|
||||
return sorted_candidates[0]
|
||||
|
||||
|
||||
def _filter_for_platform(container: str,
|
||||
registry_info: RegistryInfo,
|
||||
tag_list: List[str]) -> List[str] :
|
||||
filtered_tags = []
|
||||
this_machine = platform.machine()
|
||||
# Translate between Python platform names and docker
|
||||
if this_machine == "x86_64":
|
||||
this_machine = "amd64"
|
||||
if opts.o.debug:
|
||||
print(f"Python says the architecture is: {this_machine}")
|
||||
docker = DockerClient()
|
||||
docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token)
|
||||
for tag in tag_list:
|
||||
remote_tag = f"{registry_info.registry}/{container}:{tag}"
|
||||
manifest_cmd = ExtendedManifestCLI(docker.client_config)
|
||||
manifest = manifest_cmd.inspect_verbose(remote_tag)
|
||||
if opts.o.debug:
|
||||
print(f"manifest: {manifest}")
|
||||
image_architecture = manifest["Descriptor"]["platform"]["architecture"]
|
||||
if opts.o.debug:
|
||||
print(f"image_architecture: {image_architecture}")
|
||||
if this_machine == image_architecture:
|
||||
filtered_tags.append(tag)
|
||||
if opts.o.debug:
|
||||
print(f"Tags filtered for platform: {filtered_tags}")
|
||||
return filtered_tags
|
||||
|
||||
|
||||
def _get_latest_image(container: str, registry_info: RegistryInfo):
|
||||
all_tags = _get_tags_for_container(container, registry_info)
|
||||
tags_for_platform = _filter_for_platform(container, registry_info, all_tags)
|
||||
latest_tag = _find_latest(tags_for_platform)
|
||||
return f"{container}:{latest_tag}"
|
||||
|
||||
|
||||
def _fetch_image(tag: str, registry_info: RegistryInfo):
|
||||
docker = DockerClient()
|
||||
docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token)
|
||||
remote_tag = f"{registry_info.registry}/{tag}"
|
||||
if opts.o.debug:
|
||||
print(f"Attempting to pull this image: {remote_tag}")
|
||||
docker.image.pull(remote_tag)
|
||||
|
||||
|
||||
def _exists_locally(container: str):
|
||||
docker = DockerClient()
|
||||
return docker.image.exists(_local_tag_for(container))
|
||||
|
||||
|
||||
def _add_local_tag(remote_tag: str, registry: str, local_tag: str):
|
||||
docker = DockerClient()
|
||||
docker.image.tag(f"{registry}/{remote_tag}", local_tag)
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option('--include', help="only fetch these containers")
|
||||
@click.option('--exclude', help="don\'t fetch these containers")
|
||||
@click.option("--force-local-overwrite", is_flag=True, default=False, help="Overwrite a locally built image, if present")
|
||||
@click.option("--image-registry", required=True, help="Specify the image registry to fetch from")
|
||||
@click.option("--registry-username", required=True, help="Specify the image registry username")
|
||||
@click.option("--registry-token", required=True, help="Specify the image registry access token")
|
||||
@click.pass_context
|
||||
def command(ctx, include, exclude, force_local_overwrite, image_registry, registry_username, registry_token):
|
||||
'''EXPERIMENTAL: fetch the images for a stack from remote registry'''
|
||||
|
||||
registry_info = RegistryInfo(image_registry, registry_username, registry_token)
|
||||
# Generate list of target containers
|
||||
stack = ctx.obj.stack
|
||||
containers_in_scope = get_containers_in_scope(stack)
|
||||
for container in containers_in_scope:
|
||||
local_tag = _local_tag_for(container)
|
||||
if include_exclude_check(container, include, exclude):
|
||||
if opts.o.debug:
|
||||
print(f"Processing: {container}")
|
||||
# For each container, attempt to find the latest of a set of
|
||||
# images with the correct name and platform in the specified registry
|
||||
image_to_fetch = _get_latest_image(container, registry_info)
|
||||
if opts.o.debug:
|
||||
print(f"Fetching: {image_to_fetch}")
|
||||
_fetch_image(image_to_fetch, registry_info)
|
||||
# Now check if the target container already exists exists locally already
|
||||
if (_exists_locally(container)):
|
||||
if not opts.o.quiet:
|
||||
print(f"Container image {container} already exists locally")
|
||||
# if so, fail unless the user specified force-local-overwrite
|
||||
if (force_local_overwrite):
|
||||
# In that case remove the existing :local tag
|
||||
if not opts.o.quiet:
|
||||
print(f"Warning: overwriting local tag from this image: {container} because "
|
||||
"--force-local-overwrite was specified")
|
||||
else:
|
||||
if not opts.o.quiet:
|
||||
print(f"Skipping local tagging for this image: {container} because that would "
|
||||
"overwrite an existing :local tagged image, use --force-local-overwrite to do so.")
|
||||
# Tag the fetched image with the :local tag
|
||||
_add_local_tag(image_to_fetch, image_registry, local_tag)
|
||||
else:
|
||||
if opts.o.verbose:
|
||||
print(f"Excluding: {container}")
|
@ -17,7 +17,7 @@ import click
|
||||
|
||||
from stack_orchestrator.command_types import CommandOptions
|
||||
from stack_orchestrator.repos import setup_repositories
|
||||
from stack_orchestrator.build import build_containers
|
||||
from stack_orchestrator.build import build_containers, fetch_containers
|
||||
from stack_orchestrator.build import build_npms
|
||||
from stack_orchestrator.build import build_webapp
|
||||
from stack_orchestrator.deploy.webapp import (run_webapp,
|
||||
@ -52,6 +52,7 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err
|
||||
|
||||
cli.add_command(setup_repositories.command, "setup-repositories")
|
||||
cli.add_command(build_containers.command, "build-containers")
|
||||
cli.add_command(fetch_containers.command, "fetch-containers")
|
||||
cli.add_command(build_npms.command, "build-npms")
|
||||
cli.add_command(build_webapp.command, "build-webapp")
|
||||
cli.add_command(run_webapp.command, "run-webapp")
|
||||
|
Loading…
Reference in New Issue
Block a user