From 1b3a4d740d04c68e6e3132a1c2c63f40b79f1f49 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Mon, 29 Jan 2024 14:42:01 -0600 Subject: [PATCH 1/8] Minor fixes for deploying with k8s and podman. --- stack_orchestrator/build/build_containers.py | 3 ++ stack_orchestrator/deploy/k8s/deploy_k8s.py | 43 ++++++++++++-------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/stack_orchestrator/build/build_containers.py b/stack_orchestrator/build/build_containers.py index e987c504..8c734f50 100644 --- a/stack_orchestrator/build/build_containers.py +++ b/stack_orchestrator/build/build_containers.py @@ -104,6 +104,9 @@ def process_container(stack: str, build_command = os.path.join(container_build_dir, "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}" if not dry_run: + # No PATH at all causes failures with podman. + if "PATH" not in container_build_env: + container_build_env["PATH"] = os.environ["PATH"] if verbose: print(f"Executing: {build_command} with environment: {container_build_env}") build_result = subprocess.run(build_command, shell=True, env=container_build_env) diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 0a339fe9..43a2dfed 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -135,15 +135,20 @@ class K8sDeployer(Deployer): if not self.is_kind(): ingress: client.V1Ingress = self.cluster_info.get_ingress() - if opts.o.debug: - print(f"Sending this ingress: {ingress}") - ingress_resp = self.networking_api.create_namespaced_ingress( - namespace=self.k8s_namespace, - body=ingress - ) - if opts.o.debug: - print("Ingress created:") - print(f"{ingress_resp}") + if ingress: + if opts.o.debug: + print(f"Sending this ingress: {ingress}") + ingress_resp = self.networking_api.create_namespaced_ingress( + namespace=self.k8s_namespace, + body=ingress + ) + if opts.o.debug: + print("Ingress created:") + print(f"{ingress_resp}") + else: + if opts.o.debug: + print(f"No ingress configured") + def down(self, timeout, volumes): self.connect_api() @@ -198,14 +203,18 @@ class K8sDeployer(Deployer): if not self.is_kind(): ingress: client.V1Ingress = self.cluster_info.get_ingress() - if opts.o.debug: - print(f"Deleting this ingress: {ingress}") - try: - self.networking_api.delete_namespaced_ingress( - name=ingress.metadata.name, namespace=self.k8s_namespace - ) - except client.exceptions.ApiException as e: - _check_delete_exception(e) + if ingress: + if opts.o.debug: + print(f"Deleting this ingress: {ingress}") + try: + self.networking_api.delete_namespaced_ingress( + name=ingress.metadata.name, namespace=self.k8s_namespace + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) + else: + if opts.o.debug: + print(f"No ingress to delete") if self.is_kind(): # Destroy the kind cluster -- 2.45.2 From 6d6ca2a88323dc2fed077c139af3793a589aec86 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 30 Jan 2024 01:05:03 -0600 Subject: [PATCH 2/8] ConfigMap support --- .../deploy/deployment_create.py | 56 +++++++++++++++++-- stack_orchestrator/deploy/images.py | 8 ++- stack_orchestrator/deploy/k8s/cluster_info.py | 56 +++++++++++++++---- stack_orchestrator/deploy/k8s/deploy_k8s.py | 32 ++++++++++- stack_orchestrator/deploy/k8s/helpers.py | 22 +++++--- stack_orchestrator/deploy/spec.py | 12 ++++ 6 files changed, 162 insertions(+), 24 deletions(-) diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index 138fc1bb..5cab108a 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -54,19 +54,44 @@ def _get_ports(stack): def _get_named_volumes(stack): # Parse the compose files looking for named volumes - named_volumes = [] + named_volumes = { + "rw": [], + "ro": [] + } parsed_stack = get_parsed_stack_config(stack) pods = get_pod_list(parsed_stack) yaml = get_yaml() + + def find_vol_usage(parsed_pod_file, vol): + ret = {} + if "services" in parsed_pod_file: + for svc_name, svc in parsed_pod_file["services"].items(): + if "volumes" in svc: + for svc_volume in svc["volumes"]: + parts = svc_volume.split(":") + if parts[0] == vol: + ret[svc_name] = { + "volume": parts[0], + "mount": parts[1], + "options": parts[2] if len(parts) == 3 else None + } + return ret + for pod in pods: pod_file_path = get_pod_file_path(parsed_stack, pod) parsed_pod_file = yaml.load(open(pod_file_path, "r")) if "volumes" in parsed_pod_file: volumes = parsed_pod_file["volumes"] for volume in volumes.keys(): - # Volume definition looks like: - # 'laconicd-data': None - named_volumes.append(volume) + for vu in find_vol_usage(parsed_pod_file, volume).values(): + read_only = vu["options"] == "ro" + if read_only: + if vu["volume"] not in named_volumes["rw"] and vu["volume"] not in named_volumes["ro"]: + named_volumes["ro"].append(vu["volume"]) + else: + if vu["volume"] not in named_volumes["rw"]: + named_volumes["rw"].append(vu["volume"]) + return named_volumes @@ -104,6 +129,18 @@ def _fixup_pod_file(pod, spec, compose_dir): } } pod["volumes"][volume] = new_volume_spec + + # Fix up configmaps + if "configmaps" in spec: + spec_cfgmaps = spec["configmaps"] + if "volumes" in pod: + pod_volumes = pod["volumes"] + for volume in pod_volumes.keys(): + if volume in spec_cfgmaps: + volume_cfg = spec_cfgmaps[volume] + # Just make the dir (if necessary) + _create_bind_dir_if_relative(volume, volume_cfg, compose_dir) + # Fix up ports if "network" in spec and "ports" in spec["network"]: spec_ports = spec["network"]["ports"] @@ -319,9 +356,18 @@ def init_operation(deploy_command_context, stack, deployer_type, config, named_volumes = _get_named_volumes(stack) if named_volumes: volume_descriptors = {} - for named_volume in named_volumes: + configmap_descriptors = {} + for named_volume in named_volumes["rw"]: volume_descriptors[named_volume] = f"./data/{named_volume}" + for named_volume in named_volumes["ro"]: + if "k8s" in deployer_type: + if "config" in named_volume: + configmap_descriptors[named_volume] = f"./data/{named_volume}" + else: + volume_descriptors[named_volume] = f"./data/{named_volume}" spec_file_content["volumes"] = volume_descriptors + if configmap_descriptors: + spec_file_content["configmaps"] = configmap_descriptors if opts.o.debug: print(f"Creating spec file for stack: {stack} with content: {spec_file_content}") diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py index ddbb33f7..40da1ec2 100644 --- a/stack_orchestrator/deploy/images.py +++ b/stack_orchestrator/deploy/images.py @@ -31,7 +31,13 @@ def _image_needs_pushed(image: str): def remote_tag_for_image(image: str, remote_repo_url: str): # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy - (org, image_name_with_version) = image.split("/") + major_parts = image.split("/", 2) + org = None + if 2 == len(major_parts): + org = major_parts[0] + image_name_with_version = major_parts[1] + else: + image_name_with_version = major_parts[0] (image_name, image_version) = image_name_with_version.split(":") if image_version == "local": return f"{remote_repo_url}/{image_name}:deploy" diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 85fd63a8..77dcb426 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -1,4 +1,5 @@ # Copyright © 2023 Vulcanize +import sys # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -13,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import os + from kubernetes import client from typing import Any, List, Set @@ -112,9 +115,10 @@ class ClusterInfo: services = pod["services"] for service_name in services: service_info = services[service_name] - port = int(service_info["ports"][0]) - if opts.o.debug: - print(f"service port: {port}") + if "ports" in service_info: + port = int(service_info["ports"][0]) + if opts.o.debug: + print(f"service port: {port}") service = client.V1Service( metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"), spec=client.V1ServiceSpec( @@ -150,10 +154,41 @@ class ClusterInfo: result.append(pvc) return result + def get_configmaps(self): + result = [] + spec_configmaps = self.spec.get_configmaps() + named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) + for cfg_map_name, cfg_map_path in spec_configmaps.items(): + if cfg_map_name not in named_volumes: + if opts.o.debug: + print(f"{cfg_map_name} not in pod files") + continue + if not cfg_map_path.startswith("/"): + cfg_map_path = os.path.join(os.path.dirname(self.spec.file_path), cfg_map_path) + + data = {} + for f in os.listdir(cfg_map_path): + full_path = os.path.join(cfg_map_path, f) + if os.path.isfile(full_path): + data[f] = open(full_path, 'rt').read() + + spec = client.V1ConfigMap( + metadata=client.V1ObjectMeta(name=cfg_map_name, + labels={"configmap-label": cfg_map_name}), + data=data + ) + result.append(spec) + return result + def get_pvs(self): result = [] - volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) - for volume_name in volumes: + spec_volumes = self.spec.get_volumes() + named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) + for volume_name in spec_volumes: + if volume_name not in named_volumes: + if opts.o.debug: + print(f"{volume_name} not in pod files") + continue spec = client.V1PersistentVolumeSpec( storage_class_name="manual", access_modes=["ReadWriteOnce"], @@ -178,10 +213,11 @@ class ClusterInfo: container_name = service_name service_info = services[service_name] image = service_info["image"] - port = int(service_info["ports"][0]) - if opts.o.debug: - print(f"image: {image}") - print(f"service port: {port}") + if "ports" in service_info: + port = int(service_info["ports"][0]) + if opts.o.debug: + print(f"image: {image}") + print(f"service port: {port}") # Re-write the image tag for remote deployment image_to_use = remote_tag_for_image( image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image @@ -199,7 +235,7 @@ class ClusterInfo: ), ) containers.append(container) - volumes = volumes_for_pod_files(self.parsed_pod_yaml_map) + volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec) image_pull_secrets = [client.V1LocalObjectReference(name="laconic-registry")] template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels={"app": self.app_name}), diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 43a2dfed..df1dbba2 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -1,5 +1,5 @@ # Copyright © 2023 Vulcanize - +import sys # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -110,6 +110,20 @@ class K8sDeployer(Deployer): if opts.o.debug: print("PVCs created:") print(f"{pvc_resp}") + + # Figure out the ConfigMaps for this deployment + config_maps = self.cluster_info.get_configmaps() + for cfg_map in config_maps: + if opts.o.debug: + print(f"Sending this ConfigMap: {cfg_map}") + cfg_rsp = self.core_api.create_namespaced_config_map( + body=cfg_map, + namespace=self.k8s_namespace + ) + if opts.o.debug: + print("ConfigMap created:") + print(f"{cfg_rsp}") + # Process compose files into a Deployment deployment = self.cluster_info.get_deployment(image_pull_policy=None if self.is_kind() else "Always") # Create the k8s objects @@ -180,6 +194,22 @@ class K8sDeployer(Deployer): print(f"{pvc_resp}") except client.exceptions.ApiException as e: _check_delete_exception(e) + + # Figure out the ConfigMaps for this deployment + cfg_maps = self.cluster_info.get_configmaps() + for cfg_map in cfg_maps: + if opts.o.debug: + print(f"Deleting this ConfigMap: {cfg_map}") + try: + cfg_map_resp = self.core_api.delete_namespaced_config_map( + name=cfg_map.metadata.name, namespace=self.k8s_namespace + ) + if opts.o.debug: + print("ConfigMap deleted:") + print(f"{cfg_map_resp}") + except client.exceptions.ApiException as e: + _check_delete_exception(e) + deployment = self.cluster_info.get_deployment() if opts.o.debug: print(f"Deleting this deployment: {deployment}") diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index 62545dfd..fb40e49f 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -73,7 +73,7 @@ def named_volumes_from_pod_files(parsed_pod_files): parsed_pod_file = parsed_pod_files[pod] if "volumes" in parsed_pod_file: volumes = parsed_pod_file["volumes"] - for volume in volumes.keys(): + for volume, value in volumes.items(): # Volume definition looks like: # 'laconicd-data': None named_volumes.append(volume) @@ -98,22 +98,30 @@ def volume_mounts_for_service(parsed_pod_files, service): volumes = service_obj["volumes"] for mount_string in volumes: # Looks like: test-data:/data - (volume_name, mount_path) = mount_string.split(":") - volume_device = client.V1VolumeMount(mount_path=mount_path, name=volume_name) + parts = mount_string.split(":") + volume_name = parts[0] + mount_path = parts[1] + mount_options = parts[2] if len(parts) == 3 else None + volume_device = client.V1VolumeMount(mount_path=mount_path, name=volume_name, read_only="ro" == mount_options) result.append(volume_device) return result -def volumes_for_pod_files(parsed_pod_files): +def volumes_for_pod_files(parsed_pod_files, spec): result = [] for pod in parsed_pod_files: parsed_pod_file = parsed_pod_files[pod] if "volumes" in parsed_pod_file: volumes = parsed_pod_file["volumes"] for volume_name in volumes.keys(): - claim = client.V1PersistentVolumeClaimVolumeSource(claim_name=volume_name) - volume = client.V1Volume(name=volume_name, persistent_volume_claim=claim) - result.append(volume) + if volume_name in spec.get_configmaps(): + config_map = client.V1ConfigMapVolumeSource(name=volume_name) + volume = client.V1Volume(name=volume_name, config_map=config_map) + result.append(volume) + else: + claim = client.V1PersistentVolumeClaimVolumeSource(claim_name=volume_name) + volume = client.V1Volume(name=volume_name, persistent_volume_claim=claim) + result.append(volume) return result diff --git a/stack_orchestrator/deploy/spec.py b/stack_orchestrator/deploy/spec.py index c4f791bf..dd6cd107 100644 --- a/stack_orchestrator/deploy/spec.py +++ b/stack_orchestrator/deploy/spec.py @@ -22,6 +22,7 @@ from stack_orchestrator import constants class Spec: obj: typing.Any + file_path: Path def __init__(self) -> None: pass @@ -29,12 +30,23 @@ class Spec: def init_from_file(self, file_path: Path): with file_path: self.obj = get_yaml().load(open(file_path, "r")) + self.file_path = file_path def get_image_registry(self): return (self.obj[constants.image_resigtry_key] if self.obj and constants.image_resigtry_key in self.obj else None) + def get_volumes(self): + return (self.obj["volumes"] + if self.obj and "volumes" in self.obj + else {}) + + def get_configmaps(self): + return (self.obj["configmaps"] + if self.obj and "configmaps" in self.obj + else {}) + def get_http_proxy(self): return (self.obj[constants.network_key][constants.http_proxy_key] if self.obj and constants.network_key in self.obj -- 2.45.2 From a39c4015e015b2d40f15c051d4a418f1bfec06ad Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 30 Jan 2024 01:07:39 -0600 Subject: [PATCH 3/8] Refactor --- stack_orchestrator/deploy/k8s/cluster_info.py | 1 - 1 file changed, 1 deletion(-) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 77dcb426..89ebd269 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -1,5 +1,4 @@ # Copyright © 2023 Vulcanize -import sys # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by -- 2.45.2 From 1799badbd82f15abb1c70fabd769ca88129b58f6 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 30 Jan 2024 22:02:28 -0600 Subject: [PATCH 4/8] Missing ps --- .../data/container-build/cerc-nextjs-base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile index 8949c4e9..c3c12cc6 100644 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile @@ -30,7 +30,7 @@ RUN \ # [Optional] Uncomment this section to install additional OS packages. RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends jq gettext-base + && apt-get -y install --no-install-recommends jq gettext-base procps # [Optional] Uncomment if you want to install more global node modules # RUN su node -c "npm install -g " -- 2.45.2 From a9b476c3266ecf8b326b1d5c556a87fe584952cb Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 30 Jan 2024 22:37:05 -0600 Subject: [PATCH 5/8] flake8 --- stack_orchestrator/build/build_containers.py | 2 ++ .../deploy/deployment_create.py | 8 ++--- stack_orchestrator/deploy/images.py | 6 ++-- stack_orchestrator/deploy/k8s/cluster_info.py | 30 +++++++++++-------- stack_orchestrator/deploy/k8s/deploy_k8s.py | 20 ++++++------- stack_orchestrator/deploy/k8s/helpers.py | 5 ++-- 6 files changed, 40 insertions(+), 31 deletions(-) diff --git a/stack_orchestrator/build/build_containers.py b/stack_orchestrator/build/build_containers.py index 8c734f50..7b0957b0 100644 --- a/stack_orchestrator/build/build_containers.py +++ b/stack_orchestrator/build/build_containers.py @@ -33,6 +33,7 @@ from stack_orchestrator.base import get_npm_registry_url # TODO: find a place for this # epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)" + def make_container_build_env(dev_root_path: str, container_build_dir: str, debug: bool, @@ -122,6 +123,7 @@ def process_container(stack: str, else: print("Skipped") + @click.command() @click.option('--include', help="only build these containers") @click.option('--exclude', help="don\'t build these containers") diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index 5cab108a..b95386b1 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -54,7 +54,7 @@ def _get_ports(stack): def _get_named_volumes(stack): # Parse the compose files looking for named volumes - named_volumes = { + named_volumes = { "rw": [], "ro": [] } @@ -123,9 +123,9 @@ def _fixup_pod_file(pod, spec, compose_dir): _create_bind_dir_if_relative(volume, volume_spec, compose_dir) new_volume_spec = {"driver": "local", "driver_opts": { - "type": "none", - "device": volume_spec_fixedup, - "o": "bind" + "type": "none", + "device": volume_spec_fixedup, + "o": "bind" } } pod["volumes"][volume] = new_volume_spec diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py index 40da1ec2..7d85a8a0 100644 --- a/stack_orchestrator/deploy/images.py +++ b/stack_orchestrator/deploy/images.py @@ -34,10 +34,10 @@ def remote_tag_for_image(image: str, remote_repo_url: str): major_parts = image.split("/", 2) org = None if 2 == len(major_parts): - org = major_parts[0] - image_name_with_version = major_parts[1] + org = major_parts[0] + image_name_with_version = major_parts[1] else: - image_name_with_version = major_parts[0] + image_name_with_version = major_parts[0] (image_name, image_version) = image_name_with_version.split(":") if image_version == "local": return f"{remote_repo_url}/{image_name}:deploy" diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index fad5d7b2..5c147ad4 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -115,9 +115,9 @@ class ClusterInfo: for service_name in services: service_info = services[service_name] if "ports" in service_info: - port = int(service_info["ports"][0]) - if opts.o.debug: - print(f"service port: {port}") + port = int(service_info["ports"][0]) + if opts.o.debug: + print(f"service port: {port}") service = client.V1Service( metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"), spec=client.V1ServiceSpec( @@ -133,21 +133,27 @@ class ClusterInfo: def get_pvcs(self): result = [] - volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) + spec_volumes = self.spec.get_volumes() + named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) if opts.o.debug: - print(f"Volumes: {volumes}") - for volume_name in volumes: + print(f"Spec Volumes: {spec_volumes}") + print(f"Named Volumes: {named_volumes}") + for volume_name in spec_volumes: + if volume_name not in named_volumes: + if opts.o.debug: + print(f"{volume_name} not in pod files") + continue spec = client.V1PersistentVolumeClaimSpec( access_modes=["ReadWriteOnce"], storage_class_name="manual", resources=client.V1ResourceRequirements( requests={"storage": "2Gi"} ), - volume_name=volume_name + volume_name=f"{self.app_name}-{volume_name}" ) pvc = client.V1PersistentVolumeClaim( - metadata=client.V1ObjectMeta(name=volume_name, - labels={"volume-label": volume_name}), + metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}", + labels={"volume-label": f"{self.app_name}-{volume_name}"}), spec=spec, ) result.append(pvc) @@ -172,7 +178,7 @@ class ClusterInfo: data[f] = open(full_path, 'rt').read() spec = client.V1ConfigMap( - metadata=client.V1ObjectMeta(name=cfg_map_name, + metadata=client.V1ObjectMeta(name=f"{self.app_name}-{cfg_map_name}", labels={"configmap-label": cfg_map_name}), data=data ) @@ -195,8 +201,8 @@ class ClusterInfo: host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name)) ) pv = client.V1PersistentVolume( - metadata=client.V1ObjectMeta(name=volume_name, - labels={"volume-label": volume_name}), + metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}", + labels={"volume-label": f"{self.app_name}-{volume_name}"}), spec=spec, ) result.append(pv) diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index df1dbba2..ed8dc6b2 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -1,5 +1,4 @@ # Copyright © 2023 Vulcanize -import sys # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -13,6 +12,8 @@ import sys # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import sys + from datetime import datetime, timezone from pathlib import Path @@ -163,7 +164,6 @@ class K8sDeployer(Deployer): if opts.o.debug: print(f"No ingress configured") - def down(self, timeout, volumes): self.connect_api() # Delete the k8s objects @@ -234,14 +234,14 @@ class K8sDeployer(Deployer): if not self.is_kind(): ingress: client.V1Ingress = self.cluster_info.get_ingress() if ingress: - if opts.o.debug: - print(f"Deleting this ingress: {ingress}") - try: - self.networking_api.delete_namespaced_ingress( - name=ingress.metadata.name, namespace=self.k8s_namespace - ) - except client.exceptions.ApiException as e: - _check_delete_exception(e) + if opts.o.debug: + print(f"Deleting this ingress: {ingress}") + try: + self.networking_api.delete_namespaced_ingress( + name=ingress.metadata.name, namespace=self.k8s_namespace + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) else: if opts.o.debug: print(f"No ingress to delete") diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index fb40e49f..b094acd5 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -102,7 +102,8 @@ def volume_mounts_for_service(parsed_pod_files, service): volume_name = parts[0] mount_path = parts[1] mount_options = parts[2] if len(parts) == 3 else None - volume_device = client.V1VolumeMount(mount_path=mount_path, name=volume_name, read_only="ro" == mount_options) + volume_device = client.V1VolumeMount( + mount_path=mount_path, name=volume_name, read_only="ro" == mount_options) result.append(volume_device) return result @@ -166,7 +167,7 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir): volume_definitions.append( f" - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n" f" containerPath: {get_node_pv_mount_path(volume_name)}" - ) + ) return ( "" if len(volume_definitions) == 0 else ( " extraMounts:\n" -- 2.45.2 From 3e7943b9fed1492734e0f7dca0c4c9905c71c4d9 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 30 Jan 2024 22:40:52 -0600 Subject: [PATCH 6/8] flake8 --- stack_orchestrator/deploy/images.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py index 7d85a8a0..28d3bdb8 100644 --- a/stack_orchestrator/deploy/images.py +++ b/stack_orchestrator/deploy/images.py @@ -32,12 +32,7 @@ def _image_needs_pushed(image: str): def remote_tag_for_image(image: str, remote_repo_url: str): # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy major_parts = image.split("/", 2) - org = None - if 2 == len(major_parts): - org = major_parts[0] - image_name_with_version = major_parts[1] - else: - image_name_with_version = major_parts[0] + image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0] (image_name, image_version) = image_name_with_version.split(":") if image_version == "local": return f"{remote_repo_url}/{image_name}:deploy" -- 2.45.2 From 23ce38e002b39f2b7fb247c49257f628f6aef5ab Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 30 Jan 2024 22:44:33 -0600 Subject: [PATCH 7/8] flake8 --- stack_orchestrator/deploy/k8s/deploy_k8s.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index ed8dc6b2..045d1893 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -12,8 +12,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import sys - from datetime import datetime, timezone from pathlib import Path @@ -162,9 +160,9 @@ class K8sDeployer(Deployer): print(f"{ingress_resp}") else: if opts.o.debug: - print(f"No ingress configured") + print("No ingress configured") - def down(self, timeout, volumes): + def down(self, timeout, volumes): # noqa: C901 self.connect_api() # Delete the k8s objects # Create the host-path-mounted PVs for this deployment @@ -244,7 +242,7 @@ class K8sDeployer(Deployer): _check_delete_exception(e) else: if opts.o.debug: - print(f"No ingress to delete") + print("No ingress to delete") if self.is_kind(): # Destroy the kind cluster -- 2.45.2 From a9aa687de6c8072e58b7bebae5d257f536515677 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 30 Jan 2024 23:06:00 -0600 Subject: [PATCH 8/8] Add comment --- stack_orchestrator/deploy/k8s/cluster_info.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 5c147ad4..f0e7c87b 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -168,9 +168,12 @@ class ClusterInfo: if opts.o.debug: print(f"{cfg_map_name} not in pod files") continue + if not cfg_map_path.startswith("/"): cfg_map_path = os.path.join(os.path.dirname(self.spec.file_path), cfg_map_path) + # Read in all the files at a single-level of the directory. This mimics the behavior + # of `kubectl create configmap foo --from-file=/path/to/dir` data = {} for f in os.listdir(cfg_map_path): full_path = os.path.join(cfg_map_path, f) -- 2.45.2