From 8dfca4b227af8306b781a8ced95917d3c87b3351 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 2 Jul 2024 12:52:27 +0800 Subject: [PATCH] fix stack path resolution This was broken for "build-containers" with external stacks. Now setup and build use the same functions. --- stack_orchestrator/build/build_util.py | 12 +++---- .../deploy/deployment_create.py | 4 +-- .../repos/setup_repositories.py | 23 +++---------- stack_orchestrator/util.py | 33 +++++++++---------- 4 files changed, 27 insertions(+), 45 deletions(-) diff --git a/stack_orchestrator/build/build_util.py b/stack_orchestrator/build/build_util.py index 7eb89ba9..15be1f9b 100644 --- a/stack_orchestrator/build/build_util.py +++ b/stack_orchestrator/build/build_util.py @@ -21,11 +21,6 @@ from stack_orchestrator.util import get_parsed_stack_config, warn_exit def get_containers_in_scope(stack: str): - # See: https://stackoverflow.com/a/20885799/1701505 - from stack_orchestrator import data - with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file: - all_containers = container_list_file.read().splitlines() - containers_in_scope = [] if stack: stack_config = get_parsed_stack_config(stack) @@ -33,11 +28,14 @@ def get_containers_in_scope(stack: str): warn_exit(f"stack {stack} does not define any containers") containers_in_scope = stack_config['containers'] else: - containers_in_scope = all_containers + # See: https://stackoverflow.com/a/20885799/1701505 + from stack_orchestrator import data + with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file: + containers_in_scope = container_list_file.read().splitlines() if opts.o.verbose: print(f'Containers: {containers_in_scope}') if stack: print(f"Stack: {stack}") - return containers_in_scope \ No newline at end of file + return containers_in_scope diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index ba709ed8..ac40b768 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -24,7 +24,7 @@ from secrets import token_hex import sys from stack_orchestrator import constants from stack_orchestrator.opts import opts -from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, +from stack_orchestrator.util import (get_internal_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file, resolve_config_dir) @@ -459,7 +459,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw _check_volume_definitions(parsed_spec) stack_name = parsed_spec["stack"] deployment_type = parsed_spec[constants.deploy_to_key] - stack_file = get_stack_file_path(stack_name) + stack_file = get_internal_stack_file_path(stack_name) parsed_stack = get_parsed_stack_config(stack_name) if opts.o.debug: print(f"parsed spec: {parsed_spec}") diff --git a/stack_orchestrator/repos/setup_repositories.py b/stack_orchestrator/repos/setup_repositories.py index d16cea2b..83075647 100644 --- a/stack_orchestrator/repos/setup_repositories.py +++ b/stack_orchestrator/repos/setup_repositories.py @@ -24,11 +24,8 @@ from git.exc import GitCommandError from tqdm import tqdm import click import importlib.resources -from pathlib import Path -import yaml -from stack_orchestrator.constants import stack_file_name from stack_orchestrator.opts import opts -from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit, warn_exit +from stack_orchestrator.util import get_parsed_stack_config, include_exclude_check, error_exit, warn_exit class GitProgress(git.RemoteProgress): @@ -227,20 +224,10 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches): repos_in_scope = [] if stack: - if stack_is_external(stack): - stack_file_path = Path(stack).joinpath(stack_file_name) - else: - # In order to be compatible with Python 3.8 we need to use this hack to get the path: - # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure - stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name) - if not stack_file_path.exists(): - error_exit(f"stack {stack} does not exist") - with stack_file_path: - stack_config = yaml.safe_load(open(stack_file_path, "r")) - if "repos" not in stack_config or stack_config["repos"] is None: - warn_exit(f"stack {stack} does not define any repositories") - else: - repos_in_scope = stack_config["repos"] + stack_config = get_parsed_stack_config(stack) + if "repos" not in stack_config or stack_config["repos"] is None: + warn_exit(f"stack {stack} does not define any repositories") + repos_in_scope = stack_config["repos"] else: repos_in_scope = all_repos diff --git a/stack_orchestrator/util.py b/stack_orchestrator/util.py index d4e4d32f..3d4054ac 100644 --- a/stack_orchestrator/util.py +++ b/stack_orchestrator/util.py @@ -20,6 +20,7 @@ import ruamel.yaml from pathlib import Path from dotenv import dotenv_values from typing import Mapping, Set, List +from stack_orchestrator.constants import stack_file_name def include_exclude_check(s, include, exclude): @@ -33,10 +34,10 @@ def include_exclude_check(s, include, exclude): return s not in exclude_list -def get_stack_file_path(stack): +def get_internal_stack_file_path(stack): # In order to be compatible with Python 3.8 we need to use this hack to get the path: # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure - stack_file_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack, "stack.yml") + stack_file_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack, stack_file_name) return stack_file_path @@ -52,21 +53,17 @@ def get_dev_root_path(ctx): # Caller can pass either the name of a stack, or a path to a stack file def get_parsed_stack_config(stack): - stack_file_path = stack if isinstance(stack, os.PathLike) else get_stack_file_path(stack) - try: - with stack_file_path: - stack_config = get_yaml().load(open(stack_file_path, "r")) - return stack_config - except FileNotFoundError as error: - # We try here to generate a useful diagnostic error - # First check if the stack directory is present - stack_directory = stack_file_path.parent - if os.path.exists(stack_directory): - print(f"Error: stack.yml file is missing from stack: {stack}") - else: - print(f"Error: stack: {stack} does not exist") - print(f"Exiting, error: {error}") - sys.exit(1) + if stack_is_external(stack): + stack_file_path = Path(stack).joinpath(stack_file_name) + else: + stack_file_path = get_internal_stack_file_path(stack) + if stack_file_path.exists(): + return get_yaml().load(open(stack_file_path, "r")) + # We try here to generate a useful diagnostic error + # First check if the stack directory is present + if stack_file_path.parent.exists(): + error_exit(f"stack.yml file is missing from stack: {stack}") + error_exit(f"stack {stack} does not exist") def get_pod_list(parsed_stack): @@ -87,7 +84,7 @@ def get_plugin_code_paths(stack) -> List[Path]: result: Set[Path] = set() for pod in pods: if type(pod) is str: - result.add(get_stack_file_path(stack).parent) + result.add(get_internal_stack_file_path(stack).parent) else: pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"]) result.add(Path(os.path.join(pod_root_dir, "stack")))