From 4956fab325b431193a57b5b2c35f393ed54b6277 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Wed, 7 Feb 2024 11:54:06 -0700 Subject: [PATCH 1/3] Add env vars from compose files --- .../data/compose/docker-compose-test.yml | 1 + .../cerc-test-container/run.sh | 3 +++ stack_orchestrator/deploy/k8s/cluster_info.py | 10 ++++++++-- stack_orchestrator/deploy/k8s/helpers.py | 20 +++++++++++++++++++ tests/deploy/run-deploy-test.sh | 7 +++++++ 5 files changed, 39 insertions(+), 2 deletions(-) diff --git a/stack_orchestrator/data/compose/docker-compose-test.yml b/stack_orchestrator/data/compose/docker-compose-test.yml index 50151f65..5c9e7f9b 100644 --- a/stack_orchestrator/data/compose/docker-compose-test.yml +++ b/stack_orchestrator/data/compose/docker-compose-test.yml @@ -5,6 +5,7 @@ services: environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_TEST_PARAM_1: ${CERC_TEST_PARAM_1:-FAILED} + CERC_TEST_PARAM_2: "CERC_TEST_PARAM_2_VALUE" volumes: - test-data:/data - test-config:/config:ro diff --git a/stack_orchestrator/data/container-build/cerc-test-container/run.sh b/stack_orchestrator/data/container-build/cerc-test-container/run.sh index 01fb874b..fa860951 100755 --- a/stack_orchestrator/data/container-build/cerc-test-container/run.sh +++ b/stack_orchestrator/data/container-build/cerc-test-container/run.sh @@ -17,6 +17,9 @@ fi if [ -n "$CERC_TEST_PARAM_1" ]; then echo "Test-param-1: ${CERC_TEST_PARAM_1}" fi +if [ -n "$CERC_TEST_PARAM_2" ]; then + echo "Test-param-2: ${CERC_TEST_PARAM_2}" +fi if [ -d "/config" ]; then echo "/config: EXISTS" diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 35b2b9da..8ff5e8ca 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -22,7 +22,7 @@ from stack_orchestrator.opts import opts from stack_orchestrator.util import env_var_map_from_file from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path -from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map +from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map, envs_from_compose_file, merge_envs from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment from stack_orchestrator.deploy.deploy_types import DeployEnvVars from stack_orchestrator.deploy.spec import Spec @@ -226,6 +226,12 @@ class ClusterInfo: if opts.o.debug: print(f"image: {image}") print(f"service port: {port}") + merged_envs = merge_envs( + envs_from_compose_file( + service_info["environment"]), self.environment_variables.map + ) if "environment" in service_info else self.environment_variables.map + envs = envs_from_environment_variables_map(merged_envs) + print(f"Merged envs: {envs}") # Re-write the image tag for remote deployment image_to_use = remote_tag_for_image( image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image @@ -234,7 +240,7 @@ class ClusterInfo: name=container_name, image=image_to_use, image_pull_policy=image_pull_policy, - env=envs_from_environment_variables_map(self.environment_variables.map), + env=envs, ports=[client.V1ContainerPort(container_port=port)], volume_mounts=volume_mounts, resources=client.V1ResourceRequirements( diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index e386b353..814848d4 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -214,6 +214,26 @@ def _generate_kind_port_mappings(parsed_pod_files): ) +# Note: this makes any duplicate definition in b overwrite a +def merge_envs(a: Mapping[str, str], b: Mapping[str, str]) -> Mapping[str, str]: + result = {**a, **b} + return result + + +# DANGER: implement me +def _expand_shell_vars(raw_val: str) -> str: + return raw_val + + +# TODO: handle the case where the same env var is defined in multiple places +def envs_from_compose_file(compose_file_envs: Mapping[str, str]) -> Mapping[str, str]: + result = {} + for env_var, env_val in compose_file_envs.items(): + expanded_env_val = _expand_shell_vars(env_val) + result.update({env_var: expanded_env_val}) + return result + + def envs_from_environment_variables_map(map: Mapping[str, str]) -> List[client.V1EnvVar]: result = [] for env_var, env_val in map.items(): diff --git a/tests/deploy/run-deploy-test.sh b/tests/deploy/run-deploy-test.sh index 4456c28d..763cde8b 100755 --- a/tests/deploy/run-deploy-test.sh +++ b/tests/deploy/run-deploy-test.sh @@ -124,6 +124,13 @@ else echo "deployment config test: FAILED" exit 1 fi +# Check the config variable CERC_TEST_PARAM_2 was passed correctly from the compose file +if [[ "$log_output_3" == *"Test-param-2: CERC_TEST_PARAM_2_VALUE"* ]]; then + echo "deployment compose config test: passed" +else + echo "deployment compose config test: FAILED" + exit 1 +fi # Stop and clean up $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes echo "Test passed" -- 2.45.2 From bb73ceb0f7ad6470d0551b76d2e8f02545843d7e Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 8 Feb 2024 07:15:52 -0700 Subject: [PATCH 2/3] Partially handle shell variable expansion --- stack_orchestrator/deploy/k8s/helpers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index 814848d4..1fe697e7 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -17,6 +17,7 @@ from kubernetes import client import os from pathlib import Path import subprocess +import re from typing import Set, Mapping, List from stack_orchestrator.opts import opts @@ -220,9 +221,16 @@ def merge_envs(a: Mapping[str, str], b: Mapping[str, str]) -> Mapping[str, str]: return result -# DANGER: implement me def _expand_shell_vars(raw_val: str) -> str: - return raw_val + # could be: or ${} or ${:-} + # TODO: implement support for variable substitution and default values + # if raw_val is like ${} print a warning and substitute an empty string + # otherwise return raw_val + match = re.search(r"^\$\{(.*)\}$", raw_val) + if match: + print(f"WARNING: found unimplemented environment variable substitution: {raw_val}") + else: + return raw_val # TODO: handle the case where the same env var is defined in multiple places -- 2.45.2 From 8ea9e3c44312c9b864a5cd5f2b03b353dd58b832 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 8 Feb 2024 12:32:21 -0700 Subject: [PATCH 3/3] Update k8s and compose deploy tests for consistency --- stack_orchestrator/deploy/k8s/cluster_info.py | 3 +- tests/deploy/run-deploy-test.sh | 34 +++++++++++++++++++ tests/k8s-deploy/run-deploy-test.sh | 9 +++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 8ff5e8ca..9806584b 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -231,7 +231,8 @@ class ClusterInfo: service_info["environment"]), self.environment_variables.map ) if "environment" in service_info else self.environment_variables.map envs = envs_from_environment_variables_map(merged_envs) - print(f"Merged envs: {envs}") + if opts.o.debug: + print(f"Merged envs: {envs}") # Re-write the image tag for remote deployment image_to_use = remote_tag_for_image( image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image diff --git a/tests/deploy/run-deploy-test.sh b/tests/deploy/run-deploy-test.sh index 763cde8b..f7a008f3 100755 --- a/tests/deploy/run-deploy-test.sh +++ b/tests/deploy/run-deploy-test.sh @@ -6,6 +6,12 @@ fi # Dump environment variables for debugging echo "Environment variables:" env + +delete_cluster_exit () { + $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes + exit 1 +} + # Test basic stack-orchestrator deploy echo "Running stack-orchestrator deploy test" # Bit of a hack, test the most recent package @@ -106,6 +112,10 @@ if [ ! "$create_file_content" == "create-command-output-data" ]; then echo "deploy create test: FAILED" exit 1 fi + +# Add a config file to be picked up by the ConfigMap before starting. +echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/data/test-config/test_config + echo "deploy create output file test: passed" # Try to start the deployment $TEST_TARGET_SO deployment --dir $test_deployment_dir start @@ -131,6 +141,30 @@ else echo "deployment compose config test: FAILED" exit 1 fi + +# Check that the ConfigMap is mounted and contains the expected content. +log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) +if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then + echo "deployment ConfigMap test: passed" +else + echo "deployment ConfigMap test: FAILED" + delete_cluster_exit +fi + +# Stop then start again and check the volume was preserved +$TEST_TARGET_SO deployment --dir $test_deployment_dir stop +# Sleep a bit just in case +# sleep for longer to check if that's why the subsequent create cluster fails +sleep 20 +$TEST_TARGET_SO deployment --dir $test_deployment_dir start +log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) +if [[ "$log_output_5" == *"Filesystem is old"* ]]; then + echo "Retain volumes test: passed" +else + echo "Retain volumes test: FAILED" + delete_cluster_exit +fi + # Stop and clean up $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes echo "Test passed" diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh index 04008217..4b17ae49 100755 --- a/tests/k8s-deploy/run-deploy-test.sh +++ b/tests/k8s-deploy/run-deploy-test.sh @@ -114,6 +114,7 @@ else echo "deployment logs test: FAILED" delete_cluster_exit fi + # Check the config variable CERC_TEST_PARAM_1 was passed correctly if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then echo "deployment config test: passed" @@ -122,6 +123,14 @@ else delete_cluster_exit fi +# Check the config variable CERC_TEST_PARAM_2 was passed correctly from the compose file +if [[ "$log_output_3" == *"Test-param-2: CERC_TEST_PARAM_2_VALUE"* ]]; then + echo "deployment compose config test: passed" +else + echo "deployment compose config test: FAILED" + exit 1 +fi + # Check that the ConfigMap is mounted and contains the expected content. log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then -- 2.45.2