diff --git a/stack_orchestrator/build/build_containers.py b/stack_orchestrator/build/build_containers.py
index 2c06afcf..71debf09 100644
--- a/stack_orchestrator/build/build_containers.py
+++ b/stack_orchestrator/build/build_containers.py
@@ -25,13 +25,13 @@ import sys
from decouple import config
import subprocess
import click
-import importlib.resources
from pathlib import Path
from stack_orchestrator.opts import opts
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, error_exit, warn_exit
+from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
from stack_orchestrator.base import get_npm_registry_url
from stack_orchestrator.build.build_types import BuildContext
from stack_orchestrator.build.publish import publish_image
+from stack_orchestrator.build.build_util import get_containers_in_scope
# TODO: find a place for this
# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
@@ -149,24 +149,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag
if not image_registry:
error_exit("--image-registry must be supplied with --publish-images")
- # See: https://stackoverflow.com/a/20885799/1701505
- from stack_orchestrator import data
- with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
- all_containers = container_list_file.read().splitlines()
-
- containers_in_scope = []
- if stack:
- stack_config = get_parsed_stack_config(stack)
- if "containers" not in stack_config or stack_config["containers"] is None:
- warn_exit(f"stack {stack} does not define any containers")
- containers_in_scope = stack_config['containers']
- else:
- containers_in_scope = all_containers
-
- if opts.o.verbose:
- print(f'Containers: {containers_in_scope}')
- if stack:
- print(f"Stack: {stack}")
+ containers_in_scope = get_containers_in_scope(stack)
container_build_env = make_container_build_env(dev_root_path,
container_build_dir,
diff --git a/stack_orchestrator/build/build_util.py b/stack_orchestrator/build/build_util.py
new file mode 100644
index 00000000..7eb89ba9
--- /dev/null
+++ b/stack_orchestrator/build/build_util.py
@@ -0,0 +1,43 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+import importlib.resources
+
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import get_parsed_stack_config, warn_exit
+
+
+def get_containers_in_scope(stack: str):
+
+ # See: https://stackoverflow.com/a/20885799/1701505
+ from stack_orchestrator import data
+ with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
+ all_containers = container_list_file.read().splitlines()
+
+ containers_in_scope = []
+ if stack:
+ stack_config = get_parsed_stack_config(stack)
+ if "containers" not in stack_config or stack_config["containers"] is None:
+ warn_exit(f"stack {stack} does not define any containers")
+ containers_in_scope = stack_config['containers']
+ else:
+ containers_in_scope = all_containers
+
+ if opts.o.verbose:
+ print(f'Containers: {containers_in_scope}')
+ if stack:
+ print(f"Stack: {stack}")
+
+ return containers_in_scope
\ No newline at end of file
diff --git a/stack_orchestrator/build/fetch_containers.py b/stack_orchestrator/build/fetch_containers.py
new file mode 100644
index 00000000..66db1848
--- /dev/null
+++ b/stack_orchestrator/build/fetch_containers.py
@@ -0,0 +1,125 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+import click
+from python_on_whales import DockerClient
+import requests
+from typing import List
+
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import include_exclude_check, error_exit
+from stack_orchestrator.build.build_util import get_containers_in_scope
+
+# Experimental fetch-container command
+
+
+def _local_tag_for(container: str):
+ return f"{container}:local"
+
+# Emulate this:
+# $ curl -u "my-username:my-token" -X GET "https:///v2/cerc-io/cerc/test-container/tags/list"
+# {"name":"cerc-io/cerc/test-container","tags":["202402232130","202402232208"]}
+def _get_tags_for_container(container: str, registry: str) -> List[str]:
+ # registry looks like: git.vdb.to/cerc-io
+ registry_parts = registry.split("/")
+ url = f"https://{registry_parts[0]}/v2/{registry_parts[1]}/{container}/tags/list"
+ if opts.o.debug:
+ print(f"Fetching tags from: {url}")
+ response = requests.get(url, auth=("dboreham", "fb44b8dd36638bcf4abf478f7b1c3e18ed578f10"))
+ if response.status_code == 200:
+ tag_info = response.json()
+ if opts.o.debug:
+ print(f"Got this: {tag_info}")
+ tags_array = tag_info["tags"]
+ return tags_array
+ else:
+ error_exit(f"failed to fetch tags from image registry, status code: {response.status_code}")
+
+
+def _find_latest(candidate_tags: List[str]):
+ # HACK: return the first one for now
+ return candidate_tags[0]
+
+
+def _get_latest_image(container: str, registry: str):
+ all_tags = _get_tags_for_container(container, registry)
+ latest_tag = _find_latest(all_tags)
+ return f"{container}:{latest_tag}"
+
+
+def _fetch_image(tag: str, registry: str):
+ docker = DockerClient()
+ remote_tag = f"{registry}/{tag}"
+ if opts.o.debug:
+ print(f"Attempting to pull this image: {remote_tag}")
+ docker.image.pull(remote_tag)
+
+
+def _exists_locally(container: str):
+ docker = DockerClient()
+ return docker.image.exists(_local_tag_for(container))
+
+
+def _remove_local_tag(container: str):
+ # TODO Do we need this?
+ pass
+
+
+def _add_local_tag(remote_tag: str, registry: str, local_tag: str):
+ docker = DockerClient()
+ docker.image.tag(f"{registry}/{remote_tag}", local_tag)
+
+
+@click.command()
+@click.option('--include', help="only fetch these containers")
+@click.option('--exclude', help="don\'t fetch these containers")
+@click.option("--force-local-overwrite", is_flag=True, default=False, help="Overwrite a locally built image, if present")
+@click.option("--image-registry", required=True, help="Specify the image registry to fetch from")
+@click.pass_context
+def command(ctx, include, exclude, force_local_overwrite, image_registry):
+ '''EXPERIMENTAL: fetch the images for a stack from remote registry'''
+
+ # Generate list of target containers
+ stack = ctx.obj.stack
+ containers_in_scope = get_containers_in_scope(stack)
+ for container in containers_in_scope:
+ local_tag = _local_tag_for(container)
+ if include_exclude_check(container, include, exclude):
+ if opts.o.debug:
+ print(f"Processing: {container}")
+ # For each container, attempt to find the latest of a set of
+ # images with the correct name and platform in the specified registry
+ image_to_fetch = _get_latest_image(container, image_registry)
+ if opts.o.debug:
+ print(f"Fetching: {image_to_fetch}")
+ _fetch_image(image_to_fetch, image_registry)
+ # Now check if the target container already exists exists locally already
+ if (_exists_locally(container)):
+ if not opts.o.quiet:
+ print(f"Container image {container} already exists locally")
+ # if so, fail unless the user specified force-local-overwrite
+ if (force_local_overwrite):
+ # In that case remove the existing :local tag
+ if not opts.o.quiet:
+ print(f"Warning: removing local tag from this image: {container} because --force-local-overwrite was specified")
+ _remove_local_tag(local_tag)
+ else:
+ if not opts.o.quiet:
+ print(f"Skipping download for this image: {container}")
+ # Tag the fetched image with the :local tag
+ _add_local_tag(image_to_fetch, image_registry, local_tag)
+ else:
+ if opts.o.verbose:
+ print(f"Excluding: {container}")
diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py
index a4644412..c0a49689 100644
--- a/stack_orchestrator/main.py
+++ b/stack_orchestrator/main.py
@@ -17,7 +17,7 @@ import click
from stack_orchestrator.command_types import CommandOptions
from stack_orchestrator.repos import setup_repositories
-from stack_orchestrator.build import build_containers
+from stack_orchestrator.build import build_containers, fetch_containers
from stack_orchestrator.build import build_npms
from stack_orchestrator.build import build_webapp
from stack_orchestrator.deploy.webapp import (run_webapp,
@@ -52,6 +52,7 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err
cli.add_command(setup_repositories.command, "setup-repositories")
cli.add_command(build_containers.command, "build-containers")
+cli.add_command(fetch_containers.command, "fetch-containers")
cli.add_command(build_npms.command, "build-npms")
cli.add_command(build_webapp.command, "build-webapp")
cli.add_command(run_webapp.command, "run-webapp")