From e975eccb10b9d48d099218fc46ce1763b4859007 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 16 Feb 2024 15:24:49 -0700 Subject: [PATCH] Fix ingress in kind --- stack_orchestrator/deploy/k8s/cluster_info.py | 10 +++++----- stack_orchestrator/deploy/k8s/deploy_k8s.py | 12 +++++++----- stack_orchestrator/deploy/k8s/helpers.py | 17 +++++++++++++++-- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 925cd0f0..55393bbf 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -101,7 +101,7 @@ class ClusterInfo: ) return service - def get_ingress(self): + def get_ingress(self, use_tls=False): # No ingress for a deployment that has no http-proxy defined, for now http_proxy_info_list = self.spec.get_http_proxy() ingress = None @@ -116,7 +116,7 @@ class ClusterInfo: tls = [client.V1IngressTLS( hosts=[host_name], secret_name=f"{self.app_name}-tls" - )] + )] if use_tls else None paths = [] for route in http_proxy_info["routes"]: path = route["path"] @@ -144,15 +144,15 @@ class ClusterInfo: ) )) spec = client.V1IngressSpec( - # FIXME tls=tls, + tls=tls, rules=rules ) ingress = client.V1Ingress( metadata=client.V1ObjectMeta( name=f"{self.app_name}-ingress", annotations={ - "kubernetes.io/ingress.class": "nginx" - # FIXME "cert-manager.io/cluster-issuer": "letsencrypt-prod" + "kubernetes.io/ingress.class": "nginx", + "cert-manager.io/cluster-issuer": "letsencrypt-prod" } ), spec=spec diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 5e5e5fc5..a3855fee 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -19,7 +19,8 @@ from kubernetes import client, config from stack_orchestrator import constants from stack_orchestrator.deploy.deployer import Deployer, DeployerConfigGenerator -from stack_orchestrator.deploy.k8s.helpers import create_cluster, destroy_cluster, load_images_into_kind, install_ingress_for_kind +from stack_orchestrator.deploy.k8s.helpers import create_cluster, destroy_cluster, load_images_into_kind +from stack_orchestrator.deploy.k8s.helpers import install_ingress_for_kind, wait_for_ingress_in_kind from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, containers_in_pod, log_stream_from_string from stack_orchestrator.deploy.k8s.helpers import generate_kind_config from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo @@ -179,6 +180,8 @@ class K8sDeployer(Deployer): if self.is_kind(): # Now configure an ingress controller (not installed by default in kind) install_ingress_for_kind() + # Wait for ingress to start (deployment provisioning will fail unless this is done) + wait_for_ingress_in_kind() else: print("Dry run mode enabled, skipping k8s API connect") @@ -186,9 +189,8 @@ class K8sDeployer(Deployer): self._create_volume_data() self._create_deployment() - import time - time.sleep(40) # FIXME - ingress: client.V1Ingress = self.cluster_info.get_ingress() + # Note: at present we don't support tls for kind (and enabling tls causes errors) + ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=not self.is_kind()) if ingress: if opts.o.debug: print(f"Sending this ingress: {ingress}") @@ -286,7 +288,7 @@ class K8sDeployer(Deployer): except client.exceptions.ApiException as e: _check_delete_exception(e) - ingress: client.V1Ingress = self.cluster_info.get_ingress() + ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=not self.is_kind()) if ingress: if opts.o.debug: print(f"Deleting this ingress: {ingress}") diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index 4474c3ad..daa85cad 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -13,14 +13,14 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from kubernetes import client, utils +from kubernetes import client, utils, watch import os from pathlib import Path import subprocess import re from typing import Set, Mapping, List -from stack_orchestrator.util import get_k8s_dir +from stack_orchestrator.util import get_k8s_dir, error_exit from stack_orchestrator.opts import opts from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names from stack_orchestrator.deploy.deployer import DeployerException @@ -45,6 +45,19 @@ def destroy_cluster(name: str): _run_command(f"kind delete cluster --name {name}") +def wait_for_ingress_in_kind(): + w = watch.Watch() + core_v1 = client.CoreV1Api() + for event in w.stream(func=core_v1.list_namespaced_pod, + namespace="ingress-nginx", + label_selector="app.kubernetes.io/component=controller", + timeout_seconds=120): + if event['object'].status.container_statuses: + if event['object'].status.container_statuses[0].ready == True: + return + error_exit("ERROR: Timed out waiting for ingress to become ready") + + def install_ingress_for_kind(): api_client = client.ApiClient() ingress_install = os.path.abspath(get_k8s_dir().joinpath("components", "ingress", "ingress-nginx-kind-deploy.yaml"))