From 8be1e684e8422d16c128362b960f159b84a5a330 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 8 Feb 2024 19:41:57 +0000 Subject: [PATCH] Process environment variables defined in compose files (#736) Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/736 Co-authored-by: David Boreham Co-committed-by: David Boreham --- .../data/compose/docker-compose-test.yml | 1 + .../cerc-test-container/run.sh | 3 ++ stack_orchestrator/deploy/k8s/cluster_info.py | 11 ++++- stack_orchestrator/deploy/k8s/helpers.py | 28 +++++++++++++ tests/deploy/run-deploy-test.sh | 41 +++++++++++++++++++ tests/k8s-deploy/run-deploy-test.sh | 9 ++++ 6 files changed, 91 insertions(+), 2 deletions(-) diff --git a/stack_orchestrator/data/compose/docker-compose-test.yml b/stack_orchestrator/data/compose/docker-compose-test.yml index 50151f65..5c9e7f9b 100644 --- a/stack_orchestrator/data/compose/docker-compose-test.yml +++ b/stack_orchestrator/data/compose/docker-compose-test.yml @@ -5,6 +5,7 @@ services: environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_TEST_PARAM_1: ${CERC_TEST_PARAM_1:-FAILED} + CERC_TEST_PARAM_2: "CERC_TEST_PARAM_2_VALUE" volumes: - test-data:/data - test-config:/config:ro diff --git a/stack_orchestrator/data/container-build/cerc-test-container/run.sh b/stack_orchestrator/data/container-build/cerc-test-container/run.sh index 01fb874b..fa860951 100755 --- a/stack_orchestrator/data/container-build/cerc-test-container/run.sh +++ b/stack_orchestrator/data/container-build/cerc-test-container/run.sh @@ -17,6 +17,9 @@ fi if [ -n "$CERC_TEST_PARAM_1" ]; then echo "Test-param-1: ${CERC_TEST_PARAM_1}" fi +if [ -n "$CERC_TEST_PARAM_2" ]; then + echo "Test-param-2: ${CERC_TEST_PARAM_2}" +fi if [ -d "/config" ]; then echo "/config: EXISTS" diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 30b2ab11..17f75651 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -22,7 +22,7 @@ from stack_orchestrator.opts import opts from stack_orchestrator.util import env_var_map_from_file from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path -from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map +from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map, envs_from_compose_file, merge_envs from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment from stack_orchestrator.deploy.deploy_types import DeployEnvVars from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits @@ -263,6 +263,13 @@ class ClusterInfo: if opts.o.debug: print(f"image: {image}") print(f"service port: {port}") + merged_envs = merge_envs( + envs_from_compose_file( + service_info["environment"]), self.environment_variables.map + ) if "environment" in service_info else self.environment_variables.map + envs = envs_from_environment_variables_map(merged_envs) + if opts.o.debug: + print(f"Merged envs: {envs}") # Re-write the image tag for remote deployment image_to_use = remote_tag_for_image( image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image @@ -271,7 +278,7 @@ class ClusterInfo: name=container_name, image=image_to_use, image_pull_policy=image_pull_policy, - env=envs_from_environment_variables_map(self.environment_variables.map), + env=envs, ports=[client.V1ContainerPort(container_port=port)], volume_mounts=volume_mounts, resources=to_k8s_resource_requirements(resources), diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index e386b353..1fe697e7 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -17,6 +17,7 @@ from kubernetes import client import os from pathlib import Path import subprocess +import re from typing import Set, Mapping, List from stack_orchestrator.opts import opts @@ -214,6 +215,33 @@ def _generate_kind_port_mappings(parsed_pod_files): ) +# Note: this makes any duplicate definition in b overwrite a +def merge_envs(a: Mapping[str, str], b: Mapping[str, str]) -> Mapping[str, str]: + result = {**a, **b} + return result + + +def _expand_shell_vars(raw_val: str) -> str: + # could be: or ${} or ${:-} + # TODO: implement support for variable substitution and default values + # if raw_val is like ${} print a warning and substitute an empty string + # otherwise return raw_val + match = re.search(r"^\$\{(.*)\}$", raw_val) + if match: + print(f"WARNING: found unimplemented environment variable substitution: {raw_val}") + else: + return raw_val + + +# TODO: handle the case where the same env var is defined in multiple places +def envs_from_compose_file(compose_file_envs: Mapping[str, str]) -> Mapping[str, str]: + result = {} + for env_var, env_val in compose_file_envs.items(): + expanded_env_val = _expand_shell_vars(env_val) + result.update({env_var: expanded_env_val}) + return result + + def envs_from_environment_variables_map(map: Mapping[str, str]) -> List[client.V1EnvVar]: result = [] for env_var, env_val in map.items(): diff --git a/tests/deploy/run-deploy-test.sh b/tests/deploy/run-deploy-test.sh index 4456c28d..f7a008f3 100755 --- a/tests/deploy/run-deploy-test.sh +++ b/tests/deploy/run-deploy-test.sh @@ -6,6 +6,12 @@ fi # Dump environment variables for debugging echo "Environment variables:" env + +delete_cluster_exit () { + $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes + exit 1 +} + # Test basic stack-orchestrator deploy echo "Running stack-orchestrator deploy test" # Bit of a hack, test the most recent package @@ -106,6 +112,10 @@ if [ ! "$create_file_content" == "create-command-output-data" ]; then echo "deploy create test: FAILED" exit 1 fi + +# Add a config file to be picked up by the ConfigMap before starting. +echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/data/test-config/test_config + echo "deploy create output file test: passed" # Try to start the deployment $TEST_TARGET_SO deployment --dir $test_deployment_dir start @@ -124,6 +134,37 @@ else echo "deployment config test: FAILED" exit 1 fi +# Check the config variable CERC_TEST_PARAM_2 was passed correctly from the compose file +if [[ "$log_output_3" == *"Test-param-2: CERC_TEST_PARAM_2_VALUE"* ]]; then + echo "deployment compose config test: passed" +else + echo "deployment compose config test: FAILED" + exit 1 +fi + +# Check that the ConfigMap is mounted and contains the expected content. +log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) +if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then + echo "deployment ConfigMap test: passed" +else + echo "deployment ConfigMap test: FAILED" + delete_cluster_exit +fi + +# Stop then start again and check the volume was preserved +$TEST_TARGET_SO deployment --dir $test_deployment_dir stop +# Sleep a bit just in case +# sleep for longer to check if that's why the subsequent create cluster fails +sleep 20 +$TEST_TARGET_SO deployment --dir $test_deployment_dir start +log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) +if [[ "$log_output_5" == *"Filesystem is old"* ]]; then + echo "Retain volumes test: passed" +else + echo "Retain volumes test: FAILED" + delete_cluster_exit +fi + # Stop and clean up $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes echo "Test passed" diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh index 04008217..4b17ae49 100755 --- a/tests/k8s-deploy/run-deploy-test.sh +++ b/tests/k8s-deploy/run-deploy-test.sh @@ -114,6 +114,7 @@ else echo "deployment logs test: FAILED" delete_cluster_exit fi + # Check the config variable CERC_TEST_PARAM_1 was passed correctly if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then echo "deployment config test: passed" @@ -122,6 +123,14 @@ else delete_cluster_exit fi +# Check the config variable CERC_TEST_PARAM_2 was passed correctly from the compose file +if [[ "$log_output_3" == *"Test-param-2: CERC_TEST_PARAM_2_VALUE"* ]]; then + echo "deployment compose config test: passed" +else + echo "deployment compose config test: FAILED" + exit 1 +fi + # Check that the ConfigMap is mounted and contains the expected content. log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then