From 87bedde5cbfe00a039499026b2c4a53d8daff471 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 21 Nov 2023 16:04:36 -0700 Subject: [PATCH] Support for k8s ingress and tls (#659) --- stack_orchestrator/constants.py | 2 + .../deploy/deployment_create.py | 6 +- stack_orchestrator/deploy/k8s/cluster_info.py | 83 +++++++++++++++-- stack_orchestrator/deploy/k8s/deploy_k8s.py | 92 ++++++++++++++++--- stack_orchestrator/deploy/spec.py | 12 +++ 5 files changed, 172 insertions(+), 23 deletions(-) diff --git a/stack_orchestrator/constants.py b/stack_orchestrator/constants.py index aedc4f3c..1cff6055 100644 --- a/stack_orchestrator/constants.py +++ b/stack_orchestrator/constants.py @@ -19,6 +19,8 @@ k8s_kind_deploy_type = "k8s-kind" k8s_deploy_type = "k8s" kube_config_key = "kube-config" deploy_to_key = "deploy-to" +network_key = "network" +http_proxy_key = "http-proxy" image_resigtry_key = "image-registry" kind_config_filename = "kind-config.yml" kube_config_filename = "kubeconfig.yml" diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index e999c1df..64647ab2 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -103,8 +103,8 @@ def _fixup_pod_file(pod, spec, compose_dir): } pod["volumes"][volume] = new_volume_spec # Fix up ports - if "ports" in spec: - spec_ports = spec["ports"] + if "network" in spec and "ports" in spec["network"]: + spec_ports = spec["network"]["ports"] for container_name, container_ports in spec_ports.items(): if container_name in pod["services"]: pod["services"][container_name]["ports"] = container_ports @@ -285,7 +285,7 @@ def init(ctx, config, kube_config, image_registry, output, map_ports_to_host): print(f"Creating spec file for stack: {stack} with content: {spec_file_content}") ports = _get_mapped_ports(stack, map_ports_to_host) - spec_file_content["ports"] = ports + spec_file_content.update({"network": {"ports": ports}}) named_volumes = _get_named_volumes(stack) if named_volumes: diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index ff052bf9..a7426804 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -22,6 +22,7 @@ from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path from stack_orchestrator.deploy.k8s.helpers import env_var_map_from_file, envs_from_environment_variables_map from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment from stack_orchestrator.deploy.deploy_types import DeployEnvVars +from stack_orchestrator.deploy.spec import Spec from stack_orchestrator.deploy.images import remote_tag_for_image @@ -29,22 +30,91 @@ class ClusterInfo: parsed_pod_yaml_map: Any image_set: Set[str] = set() app_name: str = "test-app" - deployment_name: str = "test-deployment" environment_variables: DeployEnvVars - remote_image_repo: str + spec: Spec def __init__(self) -> None: pass - def int(self, pod_files: List[str], compose_env_file, remote_image_repo): + def int(self, pod_files: List[str], compose_env_file, spec: Spec): self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files) # Find the set of images in the pods self.image_set = images_for_deployment(pod_files) self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file)) - self.remote_image_repo = remote_image_repo + self.spec = spec if (opts.o.debug): print(f"Env vars: {self.environment_variables.map}") + def get_ingress(self): + # No ingress for a deployment that has no http-proxy defined, for now + http_proxy_info_list = self.spec.get_http_proxy() + ingress = None + if http_proxy_info_list: + # TODO: handle multiple definitions + http_proxy_info = http_proxy_info_list[0] + if opts.o.debug: + print(f"http-proxy: {http_proxy_info}") + # TODO: good enough parsing for webapp deployment for now + host_name = http_proxy_info["host-name"] + rules = [] + tls = [client.V1IngressTLS( + hosts=[host_name], + secret_name=f"{self.app_name}-tls" + )] + paths = [] + for route in http_proxy_info["routes"]: + path = route["path"] + proxy_to = route["proxy-to"] + if opts.o.debug: + print(f"proxy config: {path} -> {proxy_to}") + paths.append(client.V1HTTPIngressPath( + path_type="Prefix", + path=path, + backend=client.V1IngressBackend( + service=client.V1IngressServiceBackend( + # TODO: this looks wrong + name=f"{self.app_name}-service", + # TODO: pull port number from the service + port=client.V1ServiceBackendPort(number=80) + ) + ) + )) + rules.append(client.V1IngressRule( + host=host_name, + http=client.V1HTTPIngressRuleValue( + paths=paths + ) + )) + spec = client.V1IngressSpec( + tls=tls, + rules=rules + ) + ingress = client.V1Ingress( + metadata=client.V1ObjectMeta( + name=f"{self.app_name}-ingress", + annotations={ + "kubernetes.io/ingress.class": "nginx", + "cert-manager.io/cluster-issuer": "letsencrypt-prod" + } + ), + spec=spec + ) + return ingress + + def get_service(self): + service = client.V1Service( + metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"), + spec=client.V1ServiceSpec( + type="ClusterIP", + ports=[client.V1ServicePort( + port=80, + target_port=80 + )], + selector={"app": self.app_name} + ) + ) + return service + def get_pvcs(self): result = [] volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) @@ -96,7 +166,8 @@ class ClusterInfo: service_info = services[service_name] image = service_info["image"] # Re-write the image tag for remote deployment - image_to_use = remote_tag_for_image(image, self.remote_image_repo) if self.remote_image_repo is not None else image + image_to_use = remote_tag_for_image( + image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name) container = client.V1Container( name=container_name, @@ -123,7 +194,7 @@ class ClusterInfo: deployment = client.V1Deployment( api_version="apps/v1", kind="Deployment", - metadata=client.V1ObjectMeta(name=self.deployment_name), + metadata=client.V1ObjectMeta(name=f"{self.app_name}-deployment"), spec=spec, ) return deployment diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 483f64c6..8e790d10 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -23,6 +23,15 @@ from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, log_stream from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo from stack_orchestrator.opts import opts from stack_orchestrator.deploy.deployment_context import DeploymentContext +from stack_orchestrator.util import error_exit + + +def _check_delete_exception(e: client.exceptions.ApiException): + if e.status == 404: + if opts.o.debug: + print("Failed to delete object, continuing") + else: + error_exit(f"k8s api error: {e}") class K8sDeployer(Deployer): @@ -30,6 +39,7 @@ class K8sDeployer(Deployer): type: str core_api: client.CoreV1Api apps_api: client.AppsV1Api + networking_api: client.NetworkingV1Api k8s_namespace: str = "default" kind_cluster_name: str cluster_info : ClusterInfo @@ -45,7 +55,7 @@ class K8sDeployer(Deployer): self.deployment_context = deployment_context self.kind_cluster_name = compose_project_name self.cluster_info = ClusterInfo() - self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec.obj[constants.image_resigtry_key]) + self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec) if (opts.o.debug): print(f"Deployment dir: {deployment_context.deployment_dir}") print(f"Compose files: {compose_files}") @@ -60,9 +70,11 @@ class K8sDeployer(Deployer): # Get the config file and pass to load_kube_config() config.load_kube_config(config_file=self.deployment_dir.joinpath(constants.kube_config_filename).as_posix()) self.core_api = client.CoreV1Api() + self.networking_api = client.NetworkingV1Api() self.apps_api = client.AppsV1Api() def up(self, detach, services): + if self.is_kind(): # Create the kind cluster create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath(constants.kind_config_filename)) @@ -102,6 +114,26 @@ class K8sDeployer(Deployer): print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \ {deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}") + service: client.V1Service = self.cluster_info.get_service() + service_resp = self.core_api.create_namespaced_service( + namespace=self.k8s_namespace, + body=service + ) + if opts.o.debug: + print("Service created:") + print(f"{service_resp}") + + # TODO: disable ingress for kind + ingress: client.V1Ingress = self.cluster_info.get_ingress() + + ingress_resp = self.networking_api.create_namespaced_ingress( + namespace=self.k8s_namespace, + body=ingress + ) + if opts.o.debug: + print("Ingress created:") + print(f"{ingress_resp}") + def down(self, timeout, volumes): self.connect_api() # Delete the k8s objects @@ -110,28 +142,60 @@ class K8sDeployer(Deployer): for pv in pvs: if opts.o.debug: print(f"Deleting this pv: {pv}") - pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name) - if opts.o.debug: - print("PV deleted:") - print(f"{pv_resp}") + try: + pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name) + if opts.o.debug: + print("PV deleted:") + print(f"{pv_resp}") + except client.exceptions.ApiException as e: + _check_delete_exception(e) # Figure out the PVCs for this deployment pvcs = self.cluster_info.get_pvcs() for pvc in pvcs: if opts.o.debug: print(f"Deleting this pvc: {pvc}") - pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim(name=pvc.metadata.name, namespace=self.k8s_namespace) - if opts.o.debug: - print("PVCs deleted:") - print(f"{pvc_resp}") - # Process compose files into a Deployment + try: + pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim( + name=pvc.metadata.name, namespace=self.k8s_namespace + ) + if opts.o.debug: + print("PVCs deleted:") + print(f"{pvc_resp}") + except client.exceptions.ApiException as e: + _check_delete_exception(e) deployment = self.cluster_info.get_deployment() - # Create the k8s objects if opts.o.debug: print(f"Deleting this deployment: {deployment}") - self.apps_api.delete_namespaced_deployment( - name=deployment.metadata.name, namespace=self.k8s_namespace - ) + try: + self.apps_api.delete_namespaced_deployment( + name=deployment.metadata.name, namespace=self.k8s_namespace + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) + + service: client.V1Service = self.cluster_info.get_service() + if opts.o.debug: + print(f"Deleting service: {service}") + try: + self.core_api.delete_namespaced_service( + namespace=self.k8s_namespace, + name=service.metadata.name + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) + + # TODO: disable ingress for kind + ingress: client.V1Ingress = self.cluster_info.get_ingress() + if opts.o.debug: + print(f"Deleting this ingress: {ingress}") + try: + self.networking_api.delete_namespaced_ingress( + name=ingress.metadata.name, namespace=self.k8s_namespace + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) + if self.is_kind(): # Destroy the kind cluster destroy_cluster(self.kind_cluster_name) diff --git a/stack_orchestrator/deploy/spec.py b/stack_orchestrator/deploy/spec.py index 9ee893b9..c4f791bf 100644 --- a/stack_orchestrator/deploy/spec.py +++ b/stack_orchestrator/deploy/spec.py @@ -16,6 +16,7 @@ from pathlib import Path import typing from stack_orchestrator.util import get_yaml +from stack_orchestrator import constants class Spec: @@ -28,3 +29,14 @@ class Spec: def init_from_file(self, file_path: Path): with file_path: self.obj = get_yaml().load(open(file_path, "r")) + + def get_image_registry(self): + return (self.obj[constants.image_resigtry_key] + if self.obj and constants.image_resigtry_key in self.obj + else None) + + def get_http_proxy(self): + return (self.obj[constants.network_key][constants.http_proxy_key] + if self.obj and constants.network_key in self.obj + and constants.http_proxy_key in self.obj[constants.network_key] + else None)