From 1def279d269abdf2de64b501219d49b3b1df26f7 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 9 Aug 2024 02:32:06 +0000 Subject: [PATCH] Support multiple NodePorts, static NodePort mapping, and add 'replicas' spec option (#913) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NodePort example: ``` network: ports: caddy: - 1234 - 32020:2020 ``` Replicas example: ``` replicas: 2 ``` This also adds an optimization for k8s where if a directory matching the name of a configmap exists in beneath config/ in the stack, its contents will be copied into the corresponding configmap. For example: ``` # Config files in the stack ❯ ls stack-orchestrator/config/caddyconfig Caddyfile Caddyfile.one-req-per-upstream-example # ConfigMap in the spec ❯ cat foo.yml | grep config ... configmaps: caddyconfig: ./configmaps/caddyconfig # Create the deployment ❯ laconic-so --stack ~/cerc/caddy-ethcache/stack-orchestrator/stacks/caddy-ethcache deploy create --spec-file foo.yml # The files from beneath config/ have been copied to the ConfigMap directory from the spec. ❯ ls deployment-001/configmaps/caddyconfig Caddyfile Caddyfile.one-req-per-upstream-example ``` Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/913 Reviewed-by: David Boreham Co-authored-by: Thomas E Lackey Co-committed-by: Thomas E Lackey --- stack_orchestrator/constants.py | 1 + .../deploy/deployment_create.py | 7 +++ stack_orchestrator/deploy/k8s/cluster_info.py | 51 ++++++++++++------- stack_orchestrator/deploy/k8s/deploy_k8s.py | 11 ++-- stack_orchestrator/deploy/spec.py | 3 ++ 5 files changed, 50 insertions(+), 23 deletions(-) diff --git a/stack_orchestrator/constants.py b/stack_orchestrator/constants.py index bb809404..aee36ad8 100644 --- a/stack_orchestrator/constants.py +++ b/stack_orchestrator/constants.py @@ -34,5 +34,6 @@ volumes_key = "volumes" security_key = "security" annotations_key = "annotations" labels_key = "labels" +replicas_key = "replicas" kind_config_filename = "kind-config.yml" kube_config_filename = "kubeconfig.yml" diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index bdc061d5..1534dcbb 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -514,6 +514,13 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw os.mkdir(destination_script_dir) script_paths = get_pod_script_paths(parsed_stack, pod) _copy_files_to_directory(script_paths, destination_script_dir) + if parsed_spec.is_kubernetes_deployment(): + for configmap in parsed_spec.get_configmaps(): + source_config_dir = resolve_config_dir(stack_name, configmap) + if os.path.exists(source_config_dir): + destination_config_dir = deployment_dir_path.joinpath("configmaps", configmap) + copytree(source_config_dir, destination_config_dir, dirs_exist_ok=True) + # Delegate to the stack's Python code # The deploy create command doesn't require a --stack argument so we need to insert the # stack member here. diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 7c696691..05443f72 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -78,28 +78,40 @@ class ClusterInfo: if (opts.o.debug): print(f"Env vars: {self.environment_variables.map}") - def get_nodeport(self): + def get_nodeports(self): + nodeports = [] for pod_name in self.parsed_pod_yaml_map: pod = self.parsed_pod_yaml_map[pod_name] services = pod["services"] for service_name in services: service_info = services[service_name] if "ports" in service_info: - port = int(service_info["ports"][0]) - if opts.o.debug: - print(f"service port: {port}") - service = client.V1Service( - metadata=client.V1ObjectMeta(name=f"{self.app_name}-nodeport"), - spec=client.V1ServiceSpec( - type="NodePort", - ports=[client.V1ServicePort( - port=port, - target_port=port - )], - selector={"app": self.app_name} - ) - ) - return service + for raw_port in service_info["ports"]: + if opts.o.debug: + print(f"service port: {raw_port}") + if ":" in raw_port: + parts = raw_port.split(":") + if len(parts) != 2: + raise Exception(f"Invalid port definition: {raw_port}") + node_port = int(parts[0]) + pod_port = int(parts[1]) + else: + node_port = None + pod_port = int(raw_port) + service = client.V1Service( + metadata=client.V1ObjectMeta(name=f"{self.app_name}-nodeport-{pod_port}"), + spec=client.V1ServiceSpec( + type="NodePort", + ports=[client.V1ServicePort( + port=pod_port, + target_port=pod_port, + node_port=node_port + )], + selector={"app": self.app_name} + ) + ) + nodeports.append(service) + return nodeports def get_ingress(self, use_tls=False, certificate=None, cluster_issuer="letsencrypt-prod"): # No ingress for a deployment that has no http-proxy defined, for now @@ -373,9 +385,12 @@ class ClusterInfo: spec=client.V1PodSpec(containers=containers, image_pull_secrets=image_pull_secrets, volumes=volumes), ) spec = client.V1DeploymentSpec( - replicas=1, template=template, selector={ + replicas=self.spec.get_replicas(), + template=template, selector={ "matchLabels": - {"app": self.app_name}}) + {"app": self.app_name} + } + ) deployment = client.V1Deployment( api_version="apps/v1", diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 5781cd26..cbf41d1b 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -16,6 +16,7 @@ from datetime import datetime, timezone from pathlib import Path from kubernetes import client, config +from typing import List from stack_orchestrator import constants from stack_orchestrator.deploy.deployer import Deployer, DeployerConfigGenerator @@ -246,8 +247,8 @@ class K8sDeployer(Deployer): if opts.o.debug: print("No ingress configured") - nodeport: client.V1Service = self.cluster_info.get_nodeport() - if nodeport: + nodeports: List[client.V1Service] = self.cluster_info.get_nodeports() + for nodeport in nodeports: if opts.o.debug: print(f"Sending this nodeport: {nodeport}") if not opts.o.dry_run: @@ -342,10 +343,10 @@ class K8sDeployer(Deployer): if opts.o.debug: print("No ingress to delete") - nodeport: client.V1Service = self.cluster_info.get_nodeport() - if nodeport: + nodeports: List[client.V1Service] = self.cluster_info.get_nodeports() + for nodeport in nodeports: if opts.o.debug: - print(f"Deleting this nodeport: {ingress}") + print(f"Deleting this nodeport: {nodeport}") try: self.core_api.delete_namespaced_service( namespace=self.k8s_namespace, diff --git a/stack_orchestrator/deploy/spec.py b/stack_orchestrator/deploy/spec.py index cbec8ae5..e8d293e3 100644 --- a/stack_orchestrator/deploy/spec.py +++ b/stack_orchestrator/deploy/spec.py @@ -117,6 +117,9 @@ class Spec: def get_annotations(self): return self.obj.get(constants.annotations_key, {}) + def get_replicas(self): + return self.obj.get(constants.replicas_key, 1) + def get_labels(self): return self.obj.get(constants.labels_key, {})