From 62a5e1e76941db1775f2b110591cc5cb3e14a1f6 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 21 Nov 2023 06:13:03 -0700 Subject: [PATCH 1/6] Re-structure spec file --- stack_orchestrator/deploy/deployment_create.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index e999c1df..64647ab2 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -103,8 +103,8 @@ def _fixup_pod_file(pod, spec, compose_dir): } pod["volumes"][volume] = new_volume_spec # Fix up ports - if "ports" in spec: - spec_ports = spec["ports"] + if "network" in spec and "ports" in spec["network"]: + spec_ports = spec["network"]["ports"] for container_name, container_ports in spec_ports.items(): if container_name in pod["services"]: pod["services"][container_name]["ports"] = container_ports @@ -285,7 +285,7 @@ def init(ctx, config, kube_config, image_registry, output, map_ports_to_host): print(f"Creating spec file for stack: {stack} with content: {spec_file_content}") ports = _get_mapped_ports(stack, map_ports_to_host) - spec_file_content["ports"] = ports + spec_file_content.update({"network": {"ports": ports}}) named_volumes = _get_named_volumes(stack) if named_volumes: -- 2.45.2 From 4e9151318a3d70dab0f80120669bac75d3b618b0 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 21 Nov 2023 13:22:44 -0700 Subject: [PATCH 2/6] Add ingress object --- stack_orchestrator/constants.py | 2 + stack_orchestrator/deploy/k8s/cluster_info.py | 64 +++++++++++++++++-- stack_orchestrator/deploy/k8s/deploy_k8s.py | 27 +++++++- stack_orchestrator/deploy/spec.py | 12 ++++ 4 files changed, 98 insertions(+), 7 deletions(-) diff --git a/stack_orchestrator/constants.py b/stack_orchestrator/constants.py index aedc4f3c..1cff6055 100644 --- a/stack_orchestrator/constants.py +++ b/stack_orchestrator/constants.py @@ -19,6 +19,8 @@ k8s_kind_deploy_type = "k8s-kind" k8s_deploy_type = "k8s" kube_config_key = "kube-config" deploy_to_key = "deploy-to" +network_key = "network" +http_proxy_key = "http-proxy" image_resigtry_key = "image-registry" kind_config_filename = "kind-config.yml" kube_config_filename = "kubeconfig.yml" diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index ff052bf9..f8c2ce94 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -22,6 +22,7 @@ from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path from stack_orchestrator.deploy.k8s.helpers import env_var_map_from_file, envs_from_environment_variables_map from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment from stack_orchestrator.deploy.deploy_types import DeployEnvVars +from stack_orchestrator.deploy.spec import Spec from stack_orchestrator.deploy.images import remote_tag_for_image @@ -31,20 +32,74 @@ class ClusterInfo: app_name: str = "test-app" deployment_name: str = "test-deployment" environment_variables: DeployEnvVars - remote_image_repo: str + spec: Spec def __init__(self) -> None: pass - def int(self, pod_files: List[str], compose_env_file, remote_image_repo): + def int(self, pod_files: List[str], compose_env_file, spec: Spec): self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files) # Find the set of images in the pods self.image_set = images_for_deployment(pod_files) self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file)) - self.remote_image_repo = remote_image_repo + self.spec = spec if (opts.o.debug): print(f"Env vars: {self.environment_variables.map}") + def get_ingress(self): + # No ingress for a deployment that has no http-proxy defined, for now + http_proxy_info = self.spec.get_http_proxy() + ingress = None + if http_proxy_info: + if opts.o.debug: + print(f"http-proxy: {http_proxy_info}") + # TODO: good enough parsing for webapp deployment for now + host_name = http_proxy_info["host-name"] + rules = [] + tls = [client.V1IngressTLS( + hosts=[host_name], + secret_name=f"{self.app_name}-tls" + )] + paths = [] + for route in http_proxy_info["routes"]: + path = route["path"] + proxy_to = route["proxy-to"] + if opts.o.debug: + print(f"proxy config: {path} -> {proxy_to}") + paths.append(client.V1HTTPIngressPath( + path_type="Prefix", + path=path, + backend=client.V1IngressBackend( + service=client.V1IngressServiceBackend( + # TODO: this looks wrong + name=self.deployment_name, + # TODO: pull port number from the service + port=client.V1ServiceBackendPort(number=80) + ) + ) + )) + rules.append(client.V1IngressRule( + host=host_name, + http=client.V1HTTPIngressRuleValue( + paths=paths + ) + )) + spec = client.V1IngressSpec( + tls=tls, + rules=rules + ) + ingress = client.V1Ingress( + metadata=client.V1ObjectMeta( + name=f"{self.app_name}-ingress", + annotations={ + "kubernetes.io/ingress.class": "nginx", + "cert-manager.io/cluster-issuer": "letsencrypt-prod" + } + ), + spec=spec + ) + return ingress + def get_pvcs(self): result = [] volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) @@ -96,7 +151,8 @@ class ClusterInfo: service_info = services[service_name] image = service_info["image"] # Re-write the image tag for remote deployment - image_to_use = remote_tag_for_image(image, self.remote_image_repo) if self.remote_image_repo is not None else image + image_to_use = remote_tag_for_image( + image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name) container = client.V1Container( name=container_name, diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 483f64c6..2e1f6e90 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -30,6 +30,7 @@ class K8sDeployer(Deployer): type: str core_api: client.CoreV1Api apps_api: client.AppsV1Api + networking_api: client.NetworkingV1Api k8s_namespace: str = "default" kind_cluster_name: str cluster_info : ClusterInfo @@ -45,7 +46,7 @@ class K8sDeployer(Deployer): self.deployment_context = deployment_context self.kind_cluster_name = compose_project_name self.cluster_info = ClusterInfo() - self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec.obj[constants.image_resigtry_key]) + self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec) if (opts.o.debug): print(f"Deployment dir: {deployment_context.deployment_dir}") print(f"Compose files: {compose_files}") @@ -60,9 +61,11 @@ class K8sDeployer(Deployer): # Get the config file and pass to load_kube_config() config.load_kube_config(config_file=self.deployment_dir.joinpath(constants.kube_config_filename).as_posix()) self.core_api = client.CoreV1Api() + self.networking_api = client.NetworkingV1Api() self.apps_api = client.AppsV1Api() def up(self, detach, services): + if self.is_kind(): # Create the kind cluster create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath(constants.kind_config_filename)) @@ -102,6 +105,17 @@ class K8sDeployer(Deployer): print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \ {deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}") + # TODO: disable ingress for kind + ingress: client.V1Ingress = self.cluster_info.get_ingress() + + ingress_resp = self.networking_api.create_namespaced_ingress( + namespace=self.k8s_namespace, + body=ingress + ) + if opts.o.debug: + print("Ingress created:") + print(f"{ingress_resp}") + def down(self, timeout, volumes): self.connect_api() # Delete the k8s objects @@ -124,14 +138,21 @@ class K8sDeployer(Deployer): if opts.o.debug: print("PVCs deleted:") print(f"{pvc_resp}") - # Process compose files into a Deployment deployment = self.cluster_info.get_deployment() - # Create the k8s objects if opts.o.debug: print(f"Deleting this deployment: {deployment}") self.apps_api.delete_namespaced_deployment( name=deployment.metadata.name, namespace=self.k8s_namespace ) + + # TODO: disable ingress for kind + ingress: client.V1Ingress = self.cluster_info.get_ingress() + if opts.o.debug: + print(f"Deleting this ingress: {ingress}") + self.apps_api.delete_namespaced_ingress( + name=ingress.metadata.name, namespace=self.k8s_namespace + ) + if self.is_kind(): # Destroy the kind cluster destroy_cluster(self.kind_cluster_name) diff --git a/stack_orchestrator/deploy/spec.py b/stack_orchestrator/deploy/spec.py index 9ee893b9..c4f791bf 100644 --- a/stack_orchestrator/deploy/spec.py +++ b/stack_orchestrator/deploy/spec.py @@ -16,6 +16,7 @@ from pathlib import Path import typing from stack_orchestrator.util import get_yaml +from stack_orchestrator import constants class Spec: @@ -28,3 +29,14 @@ class Spec: def init_from_file(self, file_path: Path): with file_path: self.obj = get_yaml().load(open(file_path, "r")) + + def get_image_registry(self): + return (self.obj[constants.image_resigtry_key] + if self.obj and constants.image_resigtry_key in self.obj + else None) + + def get_http_proxy(self): + return (self.obj[constants.network_key][constants.http_proxy_key] + if self.obj and constants.network_key in self.obj + and constants.http_proxy_key in self.obj[constants.network_key] + else None) -- 2.45.2 From ee3d3364ba61478918fafd9d691d832edec0fad2 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 21 Nov 2023 13:28:05 -0700 Subject: [PATCH 3/6] Fix stop command --- stack_orchestrator/deploy/k8s/cluster_info.py | 6 ++++-- stack_orchestrator/deploy/k8s/deploy_k8s.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index f8c2ce94..0bfe37bf 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -48,9 +48,11 @@ class ClusterInfo: def get_ingress(self): # No ingress for a deployment that has no http-proxy defined, for now - http_proxy_info = self.spec.get_http_proxy() + http_proxy_info_list = self.spec.get_http_proxy() ingress = None - if http_proxy_info: + if http_proxy_info_list: + # TODO: handle multiple definitions + http_proxy_info = http_proxy_info_list[0] if opts.o.debug: print(f"http-proxy: {http_proxy_info}") # TODO: good enough parsing for webapp deployment for now diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 2e1f6e90..a5fa0bcb 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -149,7 +149,7 @@ class K8sDeployer(Deployer): ingress: client.V1Ingress = self.cluster_info.get_ingress() if opts.o.debug: print(f"Deleting this ingress: {ingress}") - self.apps_api.delete_namespaced_ingress( + self.networking_api.delete_namespaced_ingress( name=ingress.metadata.name, namespace=self.k8s_namespace ) -- 2.45.2 From b608a41ff1c02326945e9b57503b8631c58b82f2 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 21 Nov 2023 14:34:01 -0700 Subject: [PATCH 4/6] Add service --- stack_orchestrator/deploy/k8s/cluster_info.py | 15 +++++++++++++++ stack_orchestrator/deploy/k8s/deploy_k8s.py | 17 +++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 0bfe37bf..56b1ed6f 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -102,6 +102,21 @@ class ClusterInfo: ) return ingress + def get_service(self): + service = client.V1Service( + metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"), + spec=client.V1ServiceSpec( + type="ClusterIP", + ports=[client.V1ServicePort( + port=80, + target_port=80 + )], + selector={"matchLabels": + {"app": self.app_name}} + ) + ) + return service + def get_pvcs(self): result = [] volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index a5fa0bcb..07d9e9ec 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -105,6 +105,15 @@ class K8sDeployer(Deployer): print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \ {deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}") + service: client.V1Service = self.cluster_info.get_service() + service_resp = self.core_api.create_namespaced_service( + namespace=self.k8s_namespace, + body=service + ) + if opts.o.debug: + print("Service created:") + print(f"{service_resp}") + # TODO: disable ingress for kind ingress: client.V1Ingress = self.cluster_info.get_ingress() @@ -145,6 +154,14 @@ class K8sDeployer(Deployer): name=deployment.metadata.name, namespace=self.k8s_namespace ) + service: client.V1Service = self.cluster_info.get_service() + if opts.o.debug: + print(f"Deleting service: {service}") + self.core_api.delete_namespaced_service( + namespace=self.k8s_namespace, + body=service + ) + # TODO: disable ingress for kind ingress: client.V1Ingress = self.cluster_info.get_ingress() if opts.o.debug: -- 2.45.2 From 99a21761bd0e59773ed1652b78c94bdec9e4dab9 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 21 Nov 2023 14:36:04 -0700 Subject: [PATCH 5/6] Fix selector --- stack_orchestrator/deploy/k8s/cluster_info.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 56b1ed6f..ea2b3683 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -111,8 +111,7 @@ class ClusterInfo: port=80, target_port=80 )], - selector={"matchLabels": - {"app": self.app_name}} + selector={"app": self.app_name} ) ) return service -- 2.45.2 From 93b32e6fd37ce8600d598e304b98feb8bcc9e6c7 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 21 Nov 2023 15:28:01 -0700 Subject: [PATCH 6/6] Implement benign delete --- stack_orchestrator/deploy/k8s/cluster_info.py | 5 +- stack_orchestrator/deploy/k8s/deploy_k8s.py | 62 +++++++++++++------ 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index ea2b3683..a7426804 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -30,7 +30,6 @@ class ClusterInfo: parsed_pod_yaml_map: Any image_set: Set[str] = set() app_name: str = "test-app" - deployment_name: str = "test-deployment" environment_variables: DeployEnvVars spec: Spec @@ -74,7 +73,7 @@ class ClusterInfo: backend=client.V1IngressBackend( service=client.V1IngressServiceBackend( # TODO: this looks wrong - name=self.deployment_name, + name=f"{self.app_name}-service", # TODO: pull port number from the service port=client.V1ServiceBackendPort(number=80) ) @@ -195,7 +194,7 @@ class ClusterInfo: deployment = client.V1Deployment( api_version="apps/v1", kind="Deployment", - metadata=client.V1ObjectMeta(name=self.deployment_name), + metadata=client.V1ObjectMeta(name=f"{self.app_name}-deployment"), spec=spec, ) return deployment diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 07d9e9ec..8e790d10 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -23,6 +23,15 @@ from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, log_stream from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo from stack_orchestrator.opts import opts from stack_orchestrator.deploy.deployment_context import DeploymentContext +from stack_orchestrator.util import error_exit + + +def _check_delete_exception(e: client.exceptions.ApiException): + if e.status == 404: + if opts.o.debug: + print("Failed to delete object, continuing") + else: + error_exit(f"k8s api error: {e}") class K8sDeployer(Deployer): @@ -133,42 +142,59 @@ class K8sDeployer(Deployer): for pv in pvs: if opts.o.debug: print(f"Deleting this pv: {pv}") - pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name) - if opts.o.debug: - print("PV deleted:") - print(f"{pv_resp}") + try: + pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name) + if opts.o.debug: + print("PV deleted:") + print(f"{pv_resp}") + except client.exceptions.ApiException as e: + _check_delete_exception(e) # Figure out the PVCs for this deployment pvcs = self.cluster_info.get_pvcs() for pvc in pvcs: if opts.o.debug: print(f"Deleting this pvc: {pvc}") - pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim(name=pvc.metadata.name, namespace=self.k8s_namespace) - if opts.o.debug: - print("PVCs deleted:") - print(f"{pvc_resp}") + try: + pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim( + name=pvc.metadata.name, namespace=self.k8s_namespace + ) + if opts.o.debug: + print("PVCs deleted:") + print(f"{pvc_resp}") + except client.exceptions.ApiException as e: + _check_delete_exception(e) deployment = self.cluster_info.get_deployment() if opts.o.debug: print(f"Deleting this deployment: {deployment}") - self.apps_api.delete_namespaced_deployment( - name=deployment.metadata.name, namespace=self.k8s_namespace - ) + try: + self.apps_api.delete_namespaced_deployment( + name=deployment.metadata.name, namespace=self.k8s_namespace + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) service: client.V1Service = self.cluster_info.get_service() if opts.o.debug: print(f"Deleting service: {service}") - self.core_api.delete_namespaced_service( - namespace=self.k8s_namespace, - body=service - ) + try: + self.core_api.delete_namespaced_service( + namespace=self.k8s_namespace, + name=service.metadata.name + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) # TODO: disable ingress for kind ingress: client.V1Ingress = self.cluster_info.get_ingress() if opts.o.debug: print(f"Deleting this ingress: {ingress}") - self.networking_api.delete_namespaced_ingress( - name=ingress.metadata.name, namespace=self.k8s_namespace - ) + try: + self.networking_api.delete_namespaced_ingress( + name=ingress.metadata.name, namespace=self.k8s_namespace + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) if self.is_kind(): # Destroy the kind cluster -- 2.45.2