diff --git a/stack_orchestrator/build/build_containers.py b/stack_orchestrator/build/build_containers.py
index 2c06afcf..71debf09 100644
--- a/stack_orchestrator/build/build_containers.py
+++ b/stack_orchestrator/build/build_containers.py
@@ -25,13 +25,13 @@ import sys
from decouple import config
import subprocess
import click
-import importlib.resources
from pathlib import Path
from stack_orchestrator.opts import opts
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, error_exit, warn_exit
+from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
from stack_orchestrator.base import get_npm_registry_url
from stack_orchestrator.build.build_types import BuildContext
from stack_orchestrator.build.publish import publish_image
+from stack_orchestrator.build.build_util import get_containers_in_scope
# TODO: find a place for this
# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
@@ -149,24 +149,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag
if not image_registry:
error_exit("--image-registry must be supplied with --publish-images")
- # See: https://stackoverflow.com/a/20885799/1701505
- from stack_orchestrator import data
- with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
- all_containers = container_list_file.read().splitlines()
-
- containers_in_scope = []
- if stack:
- stack_config = get_parsed_stack_config(stack)
- if "containers" not in stack_config or stack_config["containers"] is None:
- warn_exit(f"stack {stack} does not define any containers")
- containers_in_scope = stack_config['containers']
- else:
- containers_in_scope = all_containers
-
- if opts.o.verbose:
- print(f'Containers: {containers_in_scope}')
- if stack:
- print(f"Stack: {stack}")
+ containers_in_scope = get_containers_in_scope(stack)
container_build_env = make_container_build_env(dev_root_path,
container_build_dir,
diff --git a/stack_orchestrator/build/build_util.py b/stack_orchestrator/build/build_util.py
new file mode 100644
index 00000000..7eb89ba9
--- /dev/null
+++ b/stack_orchestrator/build/build_util.py
@@ -0,0 +1,43 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+import importlib.resources
+
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import get_parsed_stack_config, warn_exit
+
+
+def get_containers_in_scope(stack: str):
+
+ # See: https://stackoverflow.com/a/20885799/1701505
+ from stack_orchestrator import data
+ with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
+ all_containers = container_list_file.read().splitlines()
+
+ containers_in_scope = []
+ if stack:
+ stack_config = get_parsed_stack_config(stack)
+ if "containers" not in stack_config or stack_config["containers"] is None:
+ warn_exit(f"stack {stack} does not define any containers")
+ containers_in_scope = stack_config['containers']
+ else:
+ containers_in_scope = all_containers
+
+ if opts.o.verbose:
+ print(f'Containers: {containers_in_scope}')
+ if stack:
+ print(f"Stack: {stack}")
+
+ return containers_in_scope
\ No newline at end of file
diff --git a/stack_orchestrator/build/fetch_containers.py b/stack_orchestrator/build/fetch_containers.py
new file mode 100644
index 00000000..890cb94f
--- /dev/null
+++ b/stack_orchestrator/build/fetch_containers.py
@@ -0,0 +1,180 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+import click
+from dataclasses import dataclass
+import json
+import platform
+from python_on_whales import DockerClient
+from python_on_whales.components.manifest.cli_wrapper import ManifestCLI, ManifestList
+from python_on_whales.utils import run
+import requests
+from typing import List
+
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import include_exclude_check, error_exit
+from stack_orchestrator.build.build_util import get_containers_in_scope
+
+# Experimental fetch-container command
+
+
+@dataclass
+class RegistryInfo:
+ registry: str
+ registry_username: str
+ registry_token: str
+
+
+# Extending this code to support the --verbose option, cnosider contributing upstream
+# https://github.com/gabrieldemarmiesse/python-on-whales/blob/master/python_on_whales/components/manifest/cli_wrapper.py#L129
+class ExtendedManifestCLI(ManifestCLI):
+ def inspect_verbose(self, x: str) -> ManifestList:
+ """Returns a Docker manifest list object."""
+ json_str = run(self.docker_cmd + ["manifest", "inspect", "--verbose", x])
+ return json.loads(json_str)
+
+
+def _local_tag_for(container: str):
+ return f"{container}:local"
+
+
+# See: https://docker-docs.uclv.cu/registry/spec/api/
+# Emulate this:
+# $ curl -u "my-username:my-token" -X GET "https:///v2/cerc-io/cerc/test-container/tags/list"
+# {"name":"cerc-io/cerc/test-container","tags":["202402232130","202402232208"]}
+def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List[str]:
+ # registry looks like: git.vdb.to/cerc-io
+ registry_parts = registry_info.registry.split("/")
+ url = f"https://{registry_parts[0]}/v2/{registry_parts[1]}/{container}/tags/list"
+ if opts.o.debug:
+ print(f"Fetching tags from: {url}")
+ response = requests.get(url, auth=(registry_info.registry_username, registry_info.registry_token))
+ if response.status_code == 200:
+ tag_info = response.json()
+ if opts.o.debug:
+ print(f"container tags list: {tag_info}")
+ tags_array = tag_info["tags"]
+ return tags_array
+ else:
+ error_exit(f"failed to fetch tags from image registry, status code: {response.status_code}")
+
+
+def _find_latest(candidate_tags: List[str]):
+ # Lex sort should give us the latest first
+ sorted_candidates = sorted(candidate_tags)
+ if opts.o.debug:
+ print(f"sorted candidates: {sorted_candidates}")
+ return sorted_candidates[0]
+
+
+def _filter_for_platform(container: str,
+ registry_info: RegistryInfo,
+ tag_list: List[str]) -> List[str] :
+ filtered_tags = []
+ this_machine = platform.machine()
+ # Translate between Python platform names and docker
+ if this_machine == "x86_64":
+ this_machine = "amd64"
+ if opts.o.debug:
+ print(f"Python says the architecture is: {this_machine}")
+ docker = DockerClient()
+ docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token)
+ for tag in tag_list:
+ remote_tag = f"{registry_info.registry}/{container}:{tag}"
+ manifest_cmd = ExtendedManifestCLI(docker.client_config)
+ manifest = manifest_cmd.inspect_verbose(remote_tag)
+ if opts.o.debug:
+ print(f"manifest: {manifest}")
+ image_architecture = manifest["Descriptor"]["platform"]["architecture"]
+ if opts.o.debug:
+ print(f"image_architecture: {image_architecture}")
+ if this_machine == image_architecture:
+ filtered_tags.append(tag)
+ if opts.o.debug:
+ print(f"Tags filtered for platform: {filtered_tags}")
+ return filtered_tags
+
+
+def _get_latest_image(container: str, registry_info: RegistryInfo):
+ all_tags = _get_tags_for_container(container, registry_info)
+ tags_for_platform = _filter_for_platform(container, registry_info, all_tags)
+ latest_tag = _find_latest(tags_for_platform)
+ return f"{container}:{latest_tag}"
+
+
+def _fetch_image(tag: str, registry_info: RegistryInfo):
+ docker = DockerClient()
+ docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token)
+ remote_tag = f"{registry_info.registry}/{tag}"
+ if opts.o.debug:
+ print(f"Attempting to pull this image: {remote_tag}")
+ docker.image.pull(remote_tag)
+
+
+def _exists_locally(container: str):
+ docker = DockerClient()
+ return docker.image.exists(_local_tag_for(container))
+
+
+def _add_local_tag(remote_tag: str, registry: str, local_tag: str):
+ docker = DockerClient()
+ docker.image.tag(f"{registry}/{remote_tag}", local_tag)
+
+
+@click.command()
+@click.option('--include', help="only fetch these containers")
+@click.option('--exclude', help="don\'t fetch these containers")
+@click.option("--force-local-overwrite", is_flag=True, default=False, help="Overwrite a locally built image, if present")
+@click.option("--image-registry", required=True, help="Specify the image registry to fetch from")
+@click.option("--registry-username", required=True, help="Specify the image registry username")
+@click.option("--registry-token", required=True, help="Specify the image registry access token")
+@click.pass_context
+def command(ctx, include, exclude, force_local_overwrite, image_registry, registry_username, registry_token):
+ '''EXPERIMENTAL: fetch the images for a stack from remote registry'''
+
+ registry_info = RegistryInfo(image_registry, registry_username, registry_token)
+ # Generate list of target containers
+ stack = ctx.obj.stack
+ containers_in_scope = get_containers_in_scope(stack)
+ for container in containers_in_scope:
+ local_tag = _local_tag_for(container)
+ if include_exclude_check(container, include, exclude):
+ if opts.o.debug:
+ print(f"Processing: {container}")
+ # For each container, attempt to find the latest of a set of
+ # images with the correct name and platform in the specified registry
+ image_to_fetch = _get_latest_image(container, registry_info)
+ if opts.o.debug:
+ print(f"Fetching: {image_to_fetch}")
+ _fetch_image(image_to_fetch, registry_info)
+ # Now check if the target container already exists exists locally already
+ if (_exists_locally(container)):
+ if not opts.o.quiet:
+ print(f"Container image {container} already exists locally")
+ # if so, fail unless the user specified force-local-overwrite
+ if (force_local_overwrite):
+ # In that case remove the existing :local tag
+ if not opts.o.quiet:
+ print(f"Warning: overwriting local tag from this image: {container} because "
+ "--force-local-overwrite was specified")
+ else:
+ if not opts.o.quiet:
+ print(f"Skipping local tagging for this image: {container} because that would "
+ "overwrite an existing :local tagged image, use --force-local-overwrite to do so.")
+ # Tag the fetched image with the :local tag
+ _add_local_tag(image_to_fetch, image_registry, local_tag)
+ else:
+ if opts.o.verbose:
+ print(f"Excluding: {container}")
diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py
index a4644412..c0a49689 100644
--- a/stack_orchestrator/main.py
+++ b/stack_orchestrator/main.py
@@ -17,7 +17,7 @@ import click
from stack_orchestrator.command_types import CommandOptions
from stack_orchestrator.repos import setup_repositories
-from stack_orchestrator.build import build_containers
+from stack_orchestrator.build import build_containers, fetch_containers
from stack_orchestrator.build import build_npms
from stack_orchestrator.build import build_webapp
from stack_orchestrator.deploy.webapp import (run_webapp,
@@ -52,6 +52,7 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err
cli.add_command(setup_repositories.command, "setup-repositories")
cli.add_command(build_containers.command, "build-containers")
+cli.add_command(fetch_containers.command, "fetch-containers")
cli.add_command(build_npms.command, "build-npms")
cli.add_command(build_webapp.command, "build-webapp")
cli.add_command(run_webapp.command, "run-webapp")