From 36d4969b2dda4a6758b275473d9b5552faf12d28 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 9 Jul 2024 15:37:35 +0000 Subject: [PATCH] Fixes for external stack deployment (#851) Fixes - stack path resolution for `build` - external stack path resolution for deployments - "extra" config detection - `deployment ports` command - `version` command in dist or source install (without build_tag.txt) - `setup-repos`, so it won't die when an existing repo is not at a branch or exact tag Used in https://git.vdb.to/cerc-io/fixturenet-eth-stacks/pulls/14 Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/851 Reviewed-by: David Boreham --- setup.py | 4 +- stack_orchestrator/build/build_util.py | 12 +++--- stack_orchestrator/deploy/deploy.py | 24 ++++++----- stack_orchestrator/deploy/deployment.py | 7 +-- .../deploy/deployment_create.py | 11 +++-- .../repos/setup_repositories.py | 38 +++++++--------- stack_orchestrator/util.py | 43 ++++++++++--------- stack_orchestrator/version.py | 13 +++--- 8 files changed, 79 insertions(+), 73 deletions(-) diff --git a/setup.py b/setup.py index 773451f5..ace0d536 100644 --- a/setup.py +++ b/setup.py @@ -4,9 +4,11 @@ with open("README.md", "r", encoding="utf-8") as fh: long_description = fh.read() with open("requirements.txt", "r", encoding="utf-8") as fh: requirements = fh.read() +with open("stack_orchestrator/data/version.txt", "r", encoding="utf-8") as fh: + version = fh.readlines()[-1].strip(" \n") setup( name='laconic-stack-orchestrator', - version='1.0.12', + version=version, author='Cerc', author_email='info@cerc.io', license='GNU Affero General Public License', diff --git a/stack_orchestrator/build/build_util.py b/stack_orchestrator/build/build_util.py index 7eb89ba9..15be1f9b 100644 --- a/stack_orchestrator/build/build_util.py +++ b/stack_orchestrator/build/build_util.py @@ -21,11 +21,6 @@ from stack_orchestrator.util import get_parsed_stack_config, warn_exit def get_containers_in_scope(stack: str): - # See: https://stackoverflow.com/a/20885799/1701505 - from stack_orchestrator import data - with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file: - all_containers = container_list_file.read().splitlines() - containers_in_scope = [] if stack: stack_config = get_parsed_stack_config(stack) @@ -33,11 +28,14 @@ def get_containers_in_scope(stack: str): warn_exit(f"stack {stack} does not define any containers") containers_in_scope = stack_config['containers'] else: - containers_in_scope = all_containers + # See: https://stackoverflow.com/a/20885799/1701505 + from stack_orchestrator import data + with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file: + containers_in_scope = container_list_file.read().splitlines() if opts.o.verbose: print(f'Containers: {containers_in_scope}') if stack: print(f"Stack: {stack}") - return containers_in_scope \ No newline at end of file + return containers_in_scope diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index db1611f9..deb32d63 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -26,8 +26,15 @@ import click from pathlib import Path from stack_orchestrator import constants from stack_orchestrator.opts import opts -from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path -from stack_orchestrator.util import resolve_compose_file +from stack_orchestrator.util import ( + get_stack_path, + include_exclude_check, + get_parsed_stack_config, + global_options2, + get_dev_root_path, + stack_is_in_deployment, + resolve_compose_file, +) from stack_orchestrator.deploy.deployer import Deployer, DeployerException from stack_orchestrator.deploy.deployer_factory import getDeployer from stack_orchestrator.deploy.deploy_types import ClusterContext, DeployCommandContext @@ -60,6 +67,7 @@ def command(ctx, include, exclude, env_file, cluster, deploy_to): if deploy_to is None: deploy_to = "compose" + stack = get_stack_path(stack) ctx.obj = create_deploy_context(global_options2(ctx), None, stack, include, exclude, cluster, env_file, deploy_to) # Subcommand is executed now, by the magic of click @@ -274,16 +282,12 @@ def _make_default_cluster_name(deployment, compose_dir, stack, include, exclude) # stack has to be either PathLike pointing to a stack yml file, or a string with the name of a known stack def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): - dev_root_path = get_dev_root_path(ctx) - # TODO: huge hack, fix this - # If the caller passed a path for the stack file, then we know that we can get the compose files - # from the same directory - deployment = False - if isinstance(stack, os.PathLike): - compose_dir = stack.parent.joinpath("compose") - deployment = True + # TODO: hack, this should be encapsulated by the deployment context. + deployment = stack_is_in_deployment(stack) + if deployment: + compose_dir = stack.joinpath("compose") else: # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure compose_dir = Path(__file__).absolute().parent.parent.joinpath("data", "compose") diff --git a/stack_orchestrator/deploy/deployment.py b/stack_orchestrator/deploy/deployment.py index f364121f..a7fd8bb2 100644 --- a/stack_orchestrator/deploy/deployment.py +++ b/stack_orchestrator/deploy/deployment.py @@ -50,15 +50,15 @@ def command(ctx, dir): def make_deploy_context(ctx) -> DeployCommandContext: context: DeploymentContext = ctx.obj - stack_file_path = context.get_stack_file() env_file = context.get_env_file() cluster_name = context.get_cluster_id() if constants.deploy_to_key in context.spec.obj: deployment_type = context.spec.obj[constants.deploy_to_key] else: deployment_type = constants.compose_deploy_type - return create_deploy_context(ctx.parent.parent.obj, context, stack_file_path, None, None, cluster_name, env_file, - deployment_type) + stack = context.deployment_dir + return create_deploy_context(ctx.parent.parent.obj, context, stack, None, None, + cluster_name, env_file, deployment_type) @command.command() @@ -123,6 +123,7 @@ def push_images(ctx): @click.argument('extra_args', nargs=-1) # help: command: port @click.pass_context def port(ctx, extra_args): + ctx.obj = make_deploy_context(ctx) port_operation(ctx, extra_args) diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index 4e0a8e13..5f565854 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -24,7 +24,7 @@ from secrets import token_hex import sys from stack_orchestrator import constants from stack_orchestrator.opts import opts -from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, +from stack_orchestrator.util import (get_stack_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file, resolve_config_dir) @@ -238,6 +238,11 @@ def _find_extra_config_dirs(parsed_pod_file, pod): config_dir = host_path.split("/")[2] if config_dir != pod: config_dirs.add(config_dir) + for env_file in service_info.get("env_file", []): + if env_file.startswith("../config"): + config_dir = env_file.split("/")[2] + if config_dir != pod: + config_dirs.add(config_dir) return config_dirs @@ -454,7 +459,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw _check_volume_definitions(parsed_spec) stack_name = parsed_spec["stack"] deployment_type = parsed_spec[constants.deploy_to_key] - stack_file = get_stack_file_path(stack_name) + stack_file = get_stack_path(stack_name).joinpath(constants.stack_file_name) parsed_stack = get_parsed_stack_config(stack_name) if opts.o.debug: print(f"parsed spec: {parsed_spec}") @@ -467,7 +472,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw os.mkdir(deployment_dir_path) # Copy spec file and the stack file into the deployment dir copyfile(spec_file, deployment_dir_path.joinpath(constants.spec_file_name)) - copyfile(stack_file, deployment_dir_path.joinpath(os.path.basename(stack_file))) + copyfile(stack_file, deployment_dir_path.joinpath(constants.stack_file_name)) _create_deployment_file(deployment_dir_path) # Copy any config varibles from the spec file into an env file suitable for compose _write_config_file(spec_file, deployment_dir_path.joinpath(constants.config_file_name)) diff --git a/stack_orchestrator/repos/setup_repositories.py b/stack_orchestrator/repos/setup_repositories.py index 4014e183..83075647 100644 --- a/stack_orchestrator/repos/setup_repositories.py +++ b/stack_orchestrator/repos/setup_repositories.py @@ -20,14 +20,12 @@ import os import sys from decouple import config import git +from git.exc import GitCommandError from tqdm import tqdm import click import importlib.resources -from pathlib import Path -import yaml -from stack_orchestrator.constants import stack_file_name from stack_orchestrator.opts import opts -from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit, warn_exit +from stack_orchestrator.util import get_parsed_stack_config, include_exclude_check, error_exit, warn_exit class GitProgress(git.RemoteProgress): @@ -81,9 +79,13 @@ def _get_repo_current_branch_or_tag(full_filesystem_repo_path): except TypeError: # This means that the current ref is not a branch, so possibly a tag # Let's try to get the tag - current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).git.describe("--tags", "--exact-match") - # Note that git is assymetric -- the tag you told it to check out may not be the one - # you get back here (if there are multiple tags associated with the same commit) + try: + current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).git.describe("--tags", "--exact-match") + # Note that git is asymmetric -- the tag you told it to check out may not be the one + # you get back here (if there are multiple tags associated with the same commit) + except GitCommandError: + # If there is no matching branch or tag checked out, just use the current SHA + current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).commit("HEAD").hexsha return current_repo_branch_or_tag, is_branch @@ -102,7 +104,7 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully full_filesystem_repo_path ) if is_present else (None, None) if not opts.o.quiet: - present_text = f"already exists active {'branch' if is_branch else 'tag'}: {current_repo_branch_or_tag}" if is_present \ + present_text = f"already exists active {'branch' if is_branch else 'ref'}: {current_repo_branch_or_tag}" if is_present \ else 'Needs to be fetched' print(f"Checking: {full_filesystem_repo_path}: {present_text}") # Quick check that it's actually a repo @@ -120,7 +122,7 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully origin = git_repo.remotes.origin origin.pull(progress=None if opts.o.quiet else GitProgress()) else: - print("skipping pull because this repo checked out a tag") + print("skipping pull because this repo is not on a branch") else: print("(git pull skipped)") if not is_present: @@ -222,20 +224,10 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches): repos_in_scope = [] if stack: - if stack_is_external(stack): - stack_file_path = Path(stack).joinpath(stack_file_name) - else: - # In order to be compatible with Python 3.8 we need to use this hack to get the path: - # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure - stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name) - if not stack_file_path.exists(): - error_exit(f"stack {stack} does not exist") - with stack_file_path: - stack_config = yaml.safe_load(open(stack_file_path, "r")) - if "repos" not in stack_config or stack_config["repos"] is None: - warn_exit(f"stack {stack} does not define any repositories") - else: - repos_in_scope = stack_config["repos"] + stack_config = get_parsed_stack_config(stack) + if "repos" not in stack_config or stack_config["repos"] is None: + warn_exit(f"stack {stack} does not define any repositories") + repos_in_scope = stack_config["repos"] else: repos_in_scope = all_repos diff --git a/stack_orchestrator/util.py b/stack_orchestrator/util.py index d4e4d32f..d2dd0425 100644 --- a/stack_orchestrator/util.py +++ b/stack_orchestrator/util.py @@ -20,6 +20,7 @@ import ruamel.yaml from pathlib import Path from dotenv import dotenv_values from typing import Mapping, Set, List +from stack_orchestrator.constants import stack_file_name, deployment_file_name def include_exclude_check(s, include, exclude): @@ -33,11 +34,14 @@ def include_exclude_check(s, include, exclude): return s not in exclude_list -def get_stack_file_path(stack): - # In order to be compatible with Python 3.8 we need to use this hack to get the path: - # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure - stack_file_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack, "stack.yml") - return stack_file_path +def get_stack_path(stack): + if stack_is_external(stack): + stack_path = Path(stack) + else: + # In order to be compatible with Python 3.8 we need to use this hack to get the path: + # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure + stack_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack) + return stack_path def get_dev_root_path(ctx): @@ -52,21 +56,14 @@ def get_dev_root_path(ctx): # Caller can pass either the name of a stack, or a path to a stack file def get_parsed_stack_config(stack): - stack_file_path = stack if isinstance(stack, os.PathLike) else get_stack_file_path(stack) - try: - with stack_file_path: - stack_config = get_yaml().load(open(stack_file_path, "r")) - return stack_config - except FileNotFoundError as error: - # We try here to generate a useful diagnostic error - # First check if the stack directory is present - stack_directory = stack_file_path.parent - if os.path.exists(stack_directory): - print(f"Error: stack.yml file is missing from stack: {stack}") - else: - print(f"Error: stack: {stack} does not exist") - print(f"Exiting, error: {error}") - sys.exit(1) + stack_file_path = get_stack_path(stack).joinpath(stack_file_name) + if stack_file_path.exists(): + return get_yaml().load(open(stack_file_path, "r")) + # We try here to generate a useful diagnostic error + # First check if the stack directory is present + if stack_file_path.parent.exists(): + error_exit(f"stack.yml file is missing from stack: {stack}") + error_exit(f"stack {stack} does not exist") def get_pod_list(parsed_stack): @@ -87,7 +84,7 @@ def get_plugin_code_paths(stack) -> List[Path]: result: Set[Path] = set() for pod in pods: if type(pod) is str: - result.add(get_stack_file_path(stack).parent) + result.add(get_stack_path(stack)) else: pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"]) result.add(Path(os.path.join(pod_root_dir, "stack"))) @@ -199,6 +196,10 @@ def stack_is_external(stack: str): return Path(stack).exists() if stack is not None else False +def stack_is_in_deployment(stack: Path): + return stack.joinpath(deployment_file_name).exists() + + def get_yaml(): # See: https://stackoverflow.com/a/45701840/1701505 yaml = ruamel.yaml.YAML() diff --git a/stack_orchestrator/version.py b/stack_orchestrator/version.py index 68e47b44..541e5580 100644 --- a/stack_orchestrator/version.py +++ b/stack_orchestrator/version.py @@ -14,7 +14,7 @@ # along with this program. If not, see . import click -import importlib.resources +from importlib import resources, metadata @click.command() @@ -24,8 +24,11 @@ def command(ctx): # See: https://stackoverflow.com/a/20885799/1701505 from stack_orchestrator import data - with importlib.resources.open_text(data, "build_tag.txt") as version_file: - # TODO: code better version that skips comment lines - version_string = version_file.read().splitlines()[1] + if resources.is_resource(data, "build_tag.txt"): + with resources.open_text(data, "build_tag.txt") as version_file: + # TODO: code better version that skips comment lines + version_string = version_file.read().splitlines()[1] + else: + version_string = metadata.version("laconic-stack-orchestrator") + "-unknown" - print(f"Version: {version_string}") + print(version_string)