From 36bb06898392a02cfd16dffefca5070246002814 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Mon, 5 Feb 2024 14:15:11 -0600 Subject: [PATCH 1/2] Add ConfigMap test. (#726) * Add ConfigMap test. * eof * Minor tweak * Trigger test --------- Co-authored-by: David Boreham --- .gitea/workflows/triggers/test-k8s-deploy | 1 + .../data/compose/docker-compose-test.yml | 2 ++ .../cerc-test-container/run.sh | 15 +++++++++++++ stack_orchestrator/deploy/deployer_factory.py | 4 ++-- .../deploy/deployment_create.py | 2 +- stack_orchestrator/deploy/k8s/deploy_k8s.py | 5 +++-- stack_orchestrator/deploy/k8s/helpers.py | 22 ++++++++++--------- tests/k8s-deploy/run-deploy-test.sh | 18 +++++++++++++-- 8 files changed, 52 insertions(+), 17 deletions(-) diff --git a/.gitea/workflows/triggers/test-k8s-deploy b/.gitea/workflows/triggers/test-k8s-deploy index 519ff4ac..f3cf0624 100644 --- a/.gitea/workflows/triggers/test-k8s-deploy +++ b/.gitea/workflows/triggers/test-k8s-deploy @@ -1 +1,2 @@ Change this file to trigger running the test-k8s-deploy CI job +Trigger test on PR branch diff --git a/stack_orchestrator/data/compose/docker-compose-test.yml b/stack_orchestrator/data/compose/docker-compose-test.yml index 5fbf46d0..50151f65 100644 --- a/stack_orchestrator/data/compose/docker-compose-test.yml +++ b/stack_orchestrator/data/compose/docker-compose-test.yml @@ -7,8 +7,10 @@ services: CERC_TEST_PARAM_1: ${CERC_TEST_PARAM_1:-FAILED} volumes: - test-data:/data + - test-config:/config:ro ports: - "80" volumes: test-data: + test-config: diff --git a/stack_orchestrator/data/container-build/cerc-test-container/run.sh b/stack_orchestrator/data/container-build/cerc-test-container/run.sh index da0af7d5..01fb874b 100755 --- a/stack_orchestrator/data/container-build/cerc-test-container/run.sh +++ b/stack_orchestrator/data/container-build/cerc-test-container/run.sh @@ -17,5 +17,20 @@ fi if [ -n "$CERC_TEST_PARAM_1" ]; then echo "Test-param-1: ${CERC_TEST_PARAM_1}" fi + +if [ -d "/config" ]; then + echo "/config: EXISTS" + for f in /config/*; do + if [[ -f "$f" ]] || [[ -L "$f" ]]; then + echo "$f:" + cat "$f" + echo "" + echo "" + fi + done +else + echo "/config: does NOT EXIST" +fi + # Run nginx which will block here forever /usr/sbin/nginx -g "daemon off;" diff --git a/stack_orchestrator/deploy/deployer_factory.py b/stack_orchestrator/deploy/deployer_factory.py index 959c1b7a..2d01729e 100644 --- a/stack_orchestrator/deploy/deployer_factory.py +++ b/stack_orchestrator/deploy/deployer_factory.py @@ -18,11 +18,11 @@ from stack_orchestrator.deploy.k8s.deploy_k8s import K8sDeployer, K8sDeployerCon from stack_orchestrator.deploy.compose.deploy_docker import DockerDeployer, DockerDeployerConfigGenerator -def getDeployerConfigGenerator(type: str): +def getDeployerConfigGenerator(type: str, deployment_context): if type == "compose" or type is None: return DockerDeployerConfigGenerator(type) elif type == constants.k8s_deploy_type or type == constants.k8s_kind_deploy_type: - return K8sDeployerConfigGenerator(type) + return K8sDeployerConfigGenerator(type, deployment_context) else: print(f"ERROR: deploy-to {type} is not valid") diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index 44824766..bb9eab40 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -487,7 +487,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw deployment_context = DeploymentContext() deployment_context.init(deployment_dir_path) # Call the deployer to generate any deployer-specific files (e.g. for kind) - deployer_config_generator = getDeployerConfigGenerator(deployment_type) + deployer_config_generator = getDeployerConfigGenerator(deployment_type, deployment_context) # TODO: make deployment_dir_path a Path above deployer_config_generator.generate(deployment_dir_path) call_stack_deploy_create(deployment_context, [network_dir, initial_peers, deployment_command_context]) diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 045d1893..3ff50277 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -390,8 +390,9 @@ class K8sDeployer(Deployer): class K8sDeployerConfigGenerator(DeployerConfigGenerator): type: str - def __init__(self, type: str) -> None: + def __init__(self, type: str, deployment_context) -> None: self.type = type + self.deployment_context = deployment_context super().__init__() def generate(self, deployment_dir: Path): @@ -399,7 +400,7 @@ class K8sDeployerConfigGenerator(DeployerConfigGenerator): if self.type == "k8s-kind": # Check the file isn't already there # Get the config file contents - content = generate_kind_config(deployment_dir) + content = generate_kind_config(deployment_dir, self.deployment_context) if opts.o.debug: print(f"kind config is: {content}") config_file = deployment_dir.joinpath(constants.kind_config_filename) diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index 081c3e62..b0c81969 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -134,8 +134,9 @@ def _get_host_paths_for_volumes(parsed_pod_files): volumes = parsed_pod_file["volumes"] for volume_name in volumes.keys(): volume_definition = volumes[volume_name] - host_path = volume_definition["driver_opts"]["device"] - result[volume_name] = host_path + if volume_definition and "driver_opts" in volume_definition: + host_path = volume_definition["driver_opts"]["device"] + result[volume_name] = host_path return result @@ -147,7 +148,7 @@ def _make_absolute_host_path(data_mount_path: Path, deployment_dir: Path) -> Pat return Path.cwd().joinpath(deployment_dir.joinpath("compose").joinpath(data_mount_path)).resolve() -def _generate_kind_mounts(parsed_pod_files, deployment_dir): +def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context): volume_definitions = [] volume_host_path_map = _get_host_paths_for_volumes(parsed_pod_files) # Note these paths are relative to the location of the pod files (at present) @@ -163,11 +164,12 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir): volumes = service_obj["volumes"] for mount_string in volumes: # Looks like: test-data:/data - (volume_name, mount_path) = mount_string.split(":") - volume_definitions.append( - f" - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n" - f" containerPath: {get_node_pv_mount_path(volume_name)}" - ) + volume_name = mount_string.split(":")[0] + if volume_name not in deployment_context.spec.get_configmaps(): + volume_definitions.append( + f" - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n" + f" containerPath: {get_node_pv_mount_path(volume_name)}" + ) return ( "" if len(volume_definitions) == 0 else ( " extraMounts:\n" @@ -223,13 +225,13 @@ def envs_from_environment_variables_map(map: Mapping[str, str]) -> List[client.V # extraMounts: # - hostPath: /path/to/my/files # containerPath: /files -def generate_kind_config(deployment_dir: Path): +def generate_kind_config(deployment_dir: Path, deployment_context): compose_file_dir = deployment_dir.joinpath("compose") # TODO: this should come from the stack file, not this way pod_files = [p for p in compose_file_dir.iterdir() if p.is_file()] parsed_pod_files_map = parsed_pod_files_map_from_file_names(pod_files) port_mappings_yml = _generate_kind_port_mappings(parsed_pod_files_map) - mounts_yml = _generate_kind_mounts(parsed_pod_files_map, deployment_dir) + mounts_yml = _generate_kind_mounts(parsed_pod_files_map, deployment_dir, deployment_context) return ( "kind: Cluster\n" "apiVersion: kind.x-k8s.io/v1alpha4\n" diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh index de1a5f10..04008217 100755 --- a/tests/k8s-deploy/run-deploy-test.sh +++ b/tests/k8s-deploy/run-deploy-test.sh @@ -97,6 +97,10 @@ if [ ! "$create_file_content" == "create-command-output-data" ]; then echo "deploy create test: FAILED" exit 1 fi + +# Add a config file to be picked up by the ConfigMap before starting. +echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/data/test-config/test_config + echo "deploy create output file test: passed" # Try to start the deployment $TEST_TARGET_SO deployment --dir $test_deployment_dir start @@ -117,6 +121,16 @@ else echo "deployment config test: FAILED" delete_cluster_exit fi + +# Check that the ConfigMap is mounted and contains the expected content. +log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) +if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then + echo "deployment ConfigMap test: passed" +else + echo "deployment ConfigMap test: FAILED" + delete_cluster_exit +fi + # Stop then start again and check the volume was preserved $TEST_TARGET_SO deployment --dir $test_deployment_dir stop # Sleep a bit just in case @@ -125,8 +139,8 @@ sleep 20 $TEST_TARGET_SO deployment --dir $test_deployment_dir start wait_for_pods_started wait_for_log_output -log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) -if [[ "$log_output_4" == *"Filesystem is old"* ]]; then +log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) +if [[ "$log_output_5" == *"Filesystem is old"* ]]; then echo "Retain volumes test: passed" else echo "Retain volumes test: FAILED" -- 2.45.2 From f8159c51fa88c718e8643c9e8548cd78ea9456d7 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 6 Feb 2024 06:58:42 -0700 Subject: [PATCH 2/2] Implement dry run support for k8s deploy --- .../deploy/compose/deploy_docker.py | 86 ++++++++------- stack_orchestrator/deploy/deploy.py | 67 ++++------- stack_orchestrator/deploy/k8s/deploy_k8s.py | 104 ++++++++++-------- 3 files changed, 132 insertions(+), 125 deletions(-) diff --git a/stack_orchestrator/deploy/compose/deploy_docker.py b/stack_orchestrator/deploy/compose/deploy_docker.py index b2622820..ffde91c2 100644 --- a/stack_orchestrator/deploy/compose/deploy_docker.py +++ b/stack_orchestrator/deploy/compose/deploy_docker.py @@ -17,6 +17,7 @@ from pathlib import Path from python_on_whales import DockerClient, DockerException from stack_orchestrator.deploy.deployer import Deployer, DeployerException, DeployerConfigGenerator from stack_orchestrator.deploy.deployment_context import DeploymentContext +from stack_orchestrator.opts import opts class DockerDeployer(Deployer): @@ -29,60 +30,69 @@ class DockerDeployer(Deployer): self.type = type def up(self, detach, services): - try: - return self.docker.compose.up(detach=detach, services=services) - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.compose.up(detach=detach, services=services) + except DockerException as e: + raise DeployerException(e) def down(self, timeout, volumes): - try: - return self.docker.compose.down(timeout=timeout, volumes=volumes) - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.compose.down(timeout=timeout, volumes=volumes) + except DockerException as e: + raise DeployerException(e) def update(self): - try: - return self.docker.compose.restart() - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.compose.restart() + except DockerException as e: + raise DeployerException(e) def status(self): - try: - for p in self.docker.compose.ps(): - print(f"{p.name}\t{p.state.status}") - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + for p in self.docker.compose.ps(): + print(f"{p.name}\t{p.state.status}") + except DockerException as e: + raise DeployerException(e) def ps(self): - try: - return self.docker.compose.ps() - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.compose.ps() + except DockerException as e: + raise DeployerException(e) def port(self, service, private_port): - try: - return self.docker.compose.port(service=service, private_port=private_port) - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.compose.port(service=service, private_port=private_port) + except DockerException as e: + raise DeployerException(e) def execute(self, service, command, tty, envs): - try: - return self.docker.compose.execute(service=service, command=command, tty=tty, envs=envs) - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.compose.execute(service=service, command=command, tty=tty, envs=envs) + except DockerException as e: + raise DeployerException(e) def logs(self, services, tail, follow, stream): - try: - return self.docker.compose.logs(services=services, tail=tail, follow=follow, stream=stream) - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.compose.logs(services=services, tail=tail, follow=follow, stream=stream) + except DockerException as e: + raise DeployerException(e) def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False): - try: - return self.docker.run(image=image, command=command, user=user, volumes=volumes, - entrypoint=entrypoint, envs=env, detach=detach, publish=ports, publish_all=len(ports) == 0) - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.run(image=image, command=command, user=user, volumes=volumes, + entrypoint=entrypoint, envs=env, detach=detach, publish=ports, publish_all=len(ports) == 0) + except DockerException as e: + raise DeployerException(e) class DockerDeployerConfigGenerator(DeployerConfigGenerator): diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index de68154b..29afcf13 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -85,54 +85,39 @@ def create_deploy_context( def up_operation(ctx, services_list, stay_attached=False): global_context = ctx.parent.parent.obj deploy_context = ctx.obj - if not global_context.dry_run: - cluster_context = deploy_context.cluster_context - container_exec_env = _make_runtime_env(global_context) - for attr, value in container_exec_env.items(): - os.environ[attr] = value - if global_context.verbose: - print(f"Running compose up with container_exec_env: {container_exec_env}, extra_args: {services_list}") - for pre_start_command in cluster_context.pre_start_commands: - _run_command(global_context, cluster_context.cluster, pre_start_command) - deploy_context.deployer.up(detach=not stay_attached, services=services_list) - for post_start_command in cluster_context.post_start_commands: - _run_command(global_context, cluster_context.cluster, post_start_command) - _orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.deployer, container_exec_env) + cluster_context = deploy_context.cluster_context + container_exec_env = _make_runtime_env(global_context) + for attr, value in container_exec_env.items(): + os.environ[attr] = value + if global_context.verbose: + print(f"Running compose up with container_exec_env: {container_exec_env}, extra_args: {services_list}") + for pre_start_command in cluster_context.pre_start_commands: + _run_command(global_context, cluster_context.cluster, pre_start_command) + deploy_context.deployer.up(detach=not stay_attached, services=services_list) + for post_start_command in cluster_context.post_start_commands: + _run_command(global_context, cluster_context.cluster, post_start_command) + _orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.deployer, container_exec_env) def down_operation(ctx, delete_volumes, extra_args_list): - global_context = ctx.parent.parent.obj - if not global_context.dry_run: - if global_context.verbose: - print("Running compose down") - timeout_arg = None - if extra_args_list: - timeout_arg = extra_args_list[0] - # Specify shutdown timeout (default 10s) to give services enough time to shutdown gracefully - ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes) + timeout_arg = None + if extra_args_list: + timeout_arg = extra_args_list[0] + # Specify shutdown timeout (default 10s) to give services enough time to shutdown gracefully + ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes) def status_operation(ctx): - global_context = ctx.parent.parent.obj - if not global_context.dry_run: - if global_context.verbose: - print("Running compose status") - ctx.obj.deployer.status() + ctx.obj.deployer.status() def update_operation(ctx): - global_context = ctx.parent.parent.obj - if not global_context.dry_run: - if global_context.verbose: - print("Running compose update") - ctx.obj.deployer.update() + ctx.obj.deployer.update() def ps_operation(ctx): global_context = ctx.parent.parent.obj if not global_context.dry_run: - if global_context.verbose: - print("Running compose ps") container_list = ctx.obj.deployer.ps() if len(container_list) > 0: print("Running containers:") @@ -187,15 +172,11 @@ def exec_operation(ctx, extra_args): def logs_operation(ctx, tail: int, follow: bool, extra_args: str): - global_context = ctx.parent.parent.obj extra_args_list = list(extra_args) or None - if not global_context.dry_run: - if global_context.verbose: - print("Running compose logs") - services_list = extra_args_list if extra_args_list is not None else [] - logs_stream = ctx.obj.deployer.logs(services=services_list, tail=tail, follow=follow, stream=True) - for stream_type, stream_content in logs_stream: - print(stream_content.decode("utf-8"), end="") + services_list = extra_args_list if extra_args_list is not None else [] + logs_stream = ctx.obj.deployer.logs(services=services_list, tail=tail, follow=follow, stream=True) + for stream_type, stream_content in logs_stream: + print(stream_content.decode("utf-8"), end="") @command.command() @@ -463,7 +444,7 @@ def _orchestrate_cluster_config(ctx, cluster_config, deployer, container_exec_en tty=False, envs=container_exec_env) waiting_for_data = False - if ctx.debug: + if ctx.debug and not waiting_for_data: print(f"destination output: {destination_output}") diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 045d1893..929547d0 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -81,69 +81,84 @@ class K8sDeployer(Deployer): self.apps_api = client.AppsV1Api() self.custom_obj_api = client.CustomObjectsApi() - def up(self, detach, services): - - if self.is_kind(): - # Create the kind cluster - create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath(constants.kind_config_filename)) - # Ensure the referenced containers are copied into kind - load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set) - self.connect_api() - + def _create_volume_data(self): # Create the host-path-mounted PVs for this deployment pvs = self.cluster_info.get_pvs() for pv in pvs: if opts.o.debug: print(f"Sending this pv: {pv}") - pv_resp = self.core_api.create_persistent_volume(body=pv) - if opts.o.debug: - print("PVs created:") - print(f"{pv_resp}") + if not opts.o.dry_run: + pv_resp = self.core_api.create_persistent_volume(body=pv) + if opts.o.debug: + print("PVs created:") + print(f"{pv_resp}") # Figure out the PVCs for this deployment pvcs = self.cluster_info.get_pvcs() for pvc in pvcs: if opts.o.debug: print(f"Sending this pvc: {pvc}") - pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace=self.k8s_namespace) - if opts.o.debug: - print("PVCs created:") - print(f"{pvc_resp}") + + if not opts.o.dry_run: + pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace=self.k8s_namespace) + if opts.o.debug: + print("PVCs created:") + print(f"{pvc_resp}") # Figure out the ConfigMaps for this deployment config_maps = self.cluster_info.get_configmaps() for cfg_map in config_maps: if opts.o.debug: print(f"Sending this ConfigMap: {cfg_map}") - cfg_rsp = self.core_api.create_namespaced_config_map( - body=cfg_map, - namespace=self.k8s_namespace - ) - if opts.o.debug: - print("ConfigMap created:") - print(f"{cfg_rsp}") + if not opts.o.dry_run: + cfg_rsp = self.core_api.create_namespaced_config_map( + body=cfg_map, + namespace=self.k8s_namespace + ) + if opts.o.debug: + print("ConfigMap created:") + print(f"{cfg_rsp}") + def _create_deployment(self): # Process compose files into a Deployment deployment = self.cluster_info.get_deployment(image_pull_policy=None if self.is_kind() else "Always") # Create the k8s objects if opts.o.debug: print(f"Sending this deployment: {deployment}") - deployment_resp = self.apps_api.create_namespaced_deployment( - body=deployment, namespace=self.k8s_namespace - ) - if opts.o.debug: - print("Deployment created:") - print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \ - {deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}") + if not opts.o.dry_run: + deployment_resp = self.apps_api.create_namespaced_deployment( + body=deployment, namespace=self.k8s_namespace + ) + if opts.o.debug: + print("Deployment created:") + print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \ + {deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}") service: client.V1Service = self.cluster_info.get_service() - service_resp = self.core_api.create_namespaced_service( - namespace=self.k8s_namespace, - body=service - ) if opts.o.debug: - print("Service created:") - print(f"{service_resp}") + print(f"Sending this service: {service}") + if not opts.o.dry_run: + service_resp = self.core_api.create_namespaced_service( + namespace=self.k8s_namespace, + body=service + ) + if opts.o.debug: + print("Service created:") + print(f"{service_resp}") + + def up(self, detach, services): + if not opts.o.dry_run: + if self.is_kind(): + # Create the kind cluster + create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath(constants.kind_config_filename)) + # Ensure the referenced containers are copied into kind + load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set) + self.connect_api() + else: + print("Dry run mode enabled, skipping k8s API connect") + + self._create_volume_data() + self._create_deployment() if not self.is_kind(): ingress: client.V1Ingress = self.cluster_info.get_ingress() @@ -151,13 +166,14 @@ class K8sDeployer(Deployer): if ingress: if opts.o.debug: print(f"Sending this ingress: {ingress}") - ingress_resp = self.networking_api.create_namespaced_ingress( - namespace=self.k8s_namespace, - body=ingress - ) - if opts.o.debug: - print("Ingress created:") - print(f"{ingress_resp}") + if not opts.o.dry_run: + ingress_resp = self.networking_api.create_namespaced_ingress( + namespace=self.k8s_namespace, + body=ingress + ) + if opts.o.debug: + print("Ingress created:") + print(f"{ingress_resp}") else: if opts.o.debug: print("No ingress configured") -- 2.45.2