From 33f640cc3367ed6828a0d996a9b73f15f437674d Mon Sep 17 00:00:00 2001 From: David Boreham Date: Mon, 20 Nov 2023 20:17:25 -0700 Subject: [PATCH] Refactor to map image names --- .../deploy/compose/deploy_docker.py | 3 +- stack_orchestrator/deploy/deploy.py | 3 +- stack_orchestrator/deploy/deployer_factory.py | 6 +- stack_orchestrator/deploy/images.py | 11 +- stack_orchestrator/deploy/k8s/cluster_info.py | 9 +- stack_orchestrator/deploy/k8s/deploy_k8s.py | 11 +- stack_orchestrator/deploy/run_webapp.py | 118 +++++++++--------- 7 files changed, 86 insertions(+), 75 deletions(-) diff --git a/stack_orchestrator/deploy/compose/deploy_docker.py b/stack_orchestrator/deploy/compose/deploy_docker.py index fc249ebc..4b4e7426 100644 --- a/stack_orchestrator/deploy/compose/deploy_docker.py +++ b/stack_orchestrator/deploy/compose/deploy_docker.py @@ -16,13 +16,14 @@ from pathlib import Path from python_on_whales import DockerClient, DockerException from stack_orchestrator.deploy.deployer import Deployer, DeployerException, DeployerConfigGenerator +from stack_orchestrator.deploy.deployment_context import DeploymentContext class DockerDeployer(Deployer): name: str = "compose" type: str - def __init__(self, type, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None: + def __init__(self, type, deployment_context: DeploymentContext, compose_files, compose_project_name, compose_env_file) -> None: self.docker = DockerClient(compose_files=compose_files, compose_project_name=compose_project_name, compose_env_file=compose_env_file) self.type = type diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index 1bce586b..32c13a61 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -71,8 +71,7 @@ def create_deploy_context( env_file, deploy_to) -> DeployCommandContext: cluster_context = _make_cluster_context(global_context, stack, include, exclude, cluster, env_file) - deployment_dir = deployment_context.deployment_dir if deployment_context else None - deployer = getDeployer(deploy_to, deployment_dir, compose_files=cluster_context.compose_files, + deployer = getDeployer(deploy_to, deployment_context, compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster, compose_env_file=cluster_context.env_file) return DeployCommandContext(stack, cluster_context, deployer) diff --git a/stack_orchestrator/deploy/deployer_factory.py b/stack_orchestrator/deploy/deployer_factory.py index de2808c5..959c1b7a 100644 --- a/stack_orchestrator/deploy/deployer_factory.py +++ b/stack_orchestrator/deploy/deployer_factory.py @@ -27,10 +27,10 @@ def getDeployerConfigGenerator(type: str): print(f"ERROR: deploy-to {type} is not valid") -def getDeployer(type: str, deployment_dir, compose_files, compose_project_name, compose_env_file): +def getDeployer(type: str, deployment_context, compose_files, compose_project_name, compose_env_file): if type == "compose" or type is None: - return DockerDeployer(type, deployment_dir, compose_files, compose_project_name, compose_env_file) + return DockerDeployer(type, deployment_context, compose_files, compose_project_name, compose_env_file) elif type == type == constants.k8s_deploy_type or type == constants.k8s_kind_deploy_type: - return K8sDeployer(type, deployment_dir, compose_files, compose_project_name, compose_env_file) + return K8sDeployer(type, deployment_context, compose_files, compose_project_name, compose_env_file) else: print(f"ERROR: deploy-to {type} is not valid") diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py index 9d40f28c..ddbb33f7 100644 --- a/stack_orchestrator/deploy/images.py +++ b/stack_orchestrator/deploy/images.py @@ -29,11 +29,14 @@ def _image_needs_pushed(image: str): return image.endswith(":local") -def _remote_tag_for_image(image: str, remote_repo_url: str): +def remote_tag_for_image(image: str, remote_repo_url: str): # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy (org, image_name_with_version) = image.split("/") (image_name, image_version) = image_name_with_version.split(":") - return f"{remote_repo_url}/{image_name}:deploy" + if image_version == "local": + return f"{remote_repo_url}/{image_name}:deploy" + else: + return image # TODO: needs lots of error handling @@ -46,14 +49,14 @@ def push_images_operation(command_context: DeployCommandContext, deployment_cont docker = DockerClient() for image in images: if _image_needs_pushed(image): - remote_tag = _remote_tag_for_image(image, remote_repo_url) + remote_tag = remote_tag_for_image(image, remote_repo_url) if opts.o.verbose: print(f"Tagging {image} to {remote_tag}") docker.image.tag(image, remote_tag) # Run docker push commands to upload for image in images: if _image_needs_pushed(image): - remote_tag = _remote_tag_for_image(image, remote_repo_url) + remote_tag = remote_tag_for_image(image, remote_repo_url) if opts.o.verbose: print(f"Pushing image {remote_tag}") docker.image.push(remote_tag) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 627cf338..ff052bf9 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -22,6 +22,7 @@ from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path from stack_orchestrator.deploy.k8s.helpers import env_var_map_from_file, envs_from_environment_variables_map from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment from stack_orchestrator.deploy.deploy_types import DeployEnvVars +from stack_orchestrator.deploy.images import remote_tag_for_image class ClusterInfo: @@ -30,15 +31,17 @@ class ClusterInfo: app_name: str = "test-app" deployment_name: str = "test-deployment" environment_variables: DeployEnvVars + remote_image_repo: str def __init__(self) -> None: pass - def int(self, pod_files: List[str], compose_env_file): + def int(self, pod_files: List[str], compose_env_file, remote_image_repo): self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files) # Find the set of images in the pods self.image_set = images_for_deployment(pod_files) self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file)) + self.remote_image_repo = remote_image_repo if (opts.o.debug): print(f"Env vars: {self.environment_variables.map}") @@ -92,10 +95,12 @@ class ClusterInfo: container_name = service_name service_info = services[service_name] image = service_info["image"] + # Re-write the image tag for remote deployment + image_to_use = remote_tag_for_image(image, self.remote_image_repo) if self.remote_image_repo is not None else image volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name) container = client.V1Container( name=container_name, - image=image, + image=image_to_use, env=envs_from_environment_variables_map(self.environment_variables.map), ports=[client.V1ContainerPort(container_port=80)], volume_mounts=volume_mounts, diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index aea2f320..3d0ef3ff 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -22,6 +22,7 @@ from stack_orchestrator.deploy.k8s.helpers import create_cluster, destroy_cluste from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, log_stream_from_string, generate_kind_config from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo from stack_orchestrator.opts import opts +from stack_orchestrator.deploy.deployment_context import DeploymentContext class K8sDeployer(Deployer): @@ -33,19 +34,21 @@ class K8sDeployer(Deployer): kind_cluster_name: str cluster_info : ClusterInfo deployment_dir: Path + deployment_context: DeploymentContext - def __init__(self, type, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None: + def __init__(self, type, deployment_context: DeploymentContext, compose_files, compose_project_name, compose_env_file) -> None: if (opts.o.debug): - print(f"Deployment dir: {deployment_dir}") + print(f"Deployment dir: {deployment_context.deployment_dir}") print(f"Compose files: {compose_files}") print(f"Project name: {compose_project_name}") print(f"Env file: {compose_env_file}") print(f"Type: {type}") self.type = type - self.deployment_dir = deployment_dir + self.deployment_dir = deployment_context.deployment_dir + self.deployment_context = deployment_context self.kind_cluster_name = compose_project_name self.cluster_info = ClusterInfo() - self.cluster_info.int(compose_files, compose_env_file) + self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec.obj[constants.image_resigtry_key]) def connect_api(self): if self.is_kind(): diff --git a/stack_orchestrator/deploy/run_webapp.py b/stack_orchestrator/deploy/run_webapp.py index 8b1073b1..aa22acdf 100644 --- a/stack_orchestrator/deploy/run_webapp.py +++ b/stack_orchestrator/deploy/run_webapp.py @@ -1,59 +1,59 @@ -# Copyright © 2022, 2023 Vulcanize - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# Builds webapp containers - -# env vars: -# CERC_REPO_BASE_DIR defaults to ~/cerc - -# TODO: display the available list of containers; allow re-build of either all or specific containers - -import hashlib -import click - -from dotenv import dotenv_values -from stack_orchestrator.deploy.deployer_factory import getDeployer - - -@click.command() -@click.option("--image", help="image to deploy", required=True) -@click.option("--deploy-to", default="compose", help="deployment type ([Docker] 'compose' or 'k8s')") -@click.option("--env-file", help="environment file for webapp") -@click.pass_context -def command(ctx, image, deploy_to, env_file): - '''build the specified webapp container''' - - env = {} - if env_file: - env = dotenv_values(env_file) - - unique_cluster_descriptor = f"{image},{env}" - hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest() - cluster = f"laconic-webapp-{hash}" - - deployer = getDeployer(deploy_to, - deployment_dir=None, - compose_files=None, - compose_project_name=cluster, - compose_env_file=None) - - container = deployer.run(image, command=[], user=None, volumes=[], entrypoint=None, env=env, detach=True) - - # Make configurable? - webappPort = "3000/tcp" - # TODO: This assumes a Docker container object... - if webappPort in container.network_settings.ports: - mapping = container.network_settings.ports[webappPort][0] - print(f"""Image: {image}\nID: {container.id}\nURL: http://localhost:{mapping['HostPort']}""") +# Copyright © 2022, 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Builds webapp containers + +# env vars: +# CERC_REPO_BASE_DIR defaults to ~/cerc + +# TODO: display the available list of containers; allow re-build of either all or specific containers + +import hashlib +import click + +from dotenv import dotenv_values +from stack_orchestrator.deploy.deployer_factory import getDeployer + + +@click.command() +@click.option("--image", help="image to deploy", required=True) +@click.option("--deploy-to", default="compose", help="deployment type ([Docker] 'compose' or 'k8s')") +@click.option("--env-file", help="environment file for webapp") +@click.pass_context +def command(ctx, image, deploy_to, env_file): + '''build the specified webapp container''' + + env = {} + if env_file: + env = dotenv_values(env_file) + + unique_cluster_descriptor = f"{image},{env}" + hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest() + cluster = f"laconic-webapp-{hash}" + + deployer = getDeployer(deploy_to, + deployment_context=None, + compose_files=None, + compose_project_name=cluster, + compose_env_file=None) + + container = deployer.run(image, command=[], user=None, volumes=[], entrypoint=None, env=env, detach=True) + + # Make configurable? + webappPort = "3000/tcp" + # TODO: This assumes a Docker container object... + if webappPort in container.network_settings.ports: + mapping = container.network_settings.ports[webappPort][0] + print(f"""Image: {image}\nID: {container.id}\nURL: http://localhost:{mapping['HostPort']}""")