From 638fa016498a17a0ea24f3b4401459f464d208b5 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 14 Nov 2023 20:59:48 -0700 Subject: [PATCH] Support external stack file (#650) --- stack_orchestrator/build/build_containers.py | 32 +++++++++++++++---- stack_orchestrator/build/build_webapp.py | 4 +-- stack_orchestrator/constants.py | 16 ++++++++++ .../repos/setup_repositories.py | 20 ++++++++---- stack_orchestrator/util.py | 11 +++++++ 5 files changed, 68 insertions(+), 15 deletions(-) create mode 100644 stack_orchestrator/constants.py diff --git a/stack_orchestrator/build/build_containers.py b/stack_orchestrator/build/build_containers.py index 5b2748cc..e987c504 100644 --- a/stack_orchestrator/build/build_containers.py +++ b/stack_orchestrator/build/build_containers.py @@ -27,7 +27,7 @@ import subprocess import click import importlib.resources from pathlib import Path -from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config +from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external from stack_orchestrator.base import get_npm_registry_url # TODO: find a place for this @@ -58,7 +58,8 @@ def make_container_build_env(dev_root_path: str, return container_build_env -def process_container(container, +def process_container(stack: str, + container, container_build_dir: str, container_build_env: dict, dev_root_path: str, @@ -69,12 +70,29 @@ def process_container(container, ): if not quiet: print(f"Building: {container}") - build_dir = os.path.join(container_build_dir, container.replace("/", "-")) - build_script_filename = os.path.join(build_dir, "build.sh") + + default_container_tag = f"{container}:local" + container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag}) + + # Check if this is in an external stack + if stack_is_external(stack): + container_parent_dir = Path(stack).joinpath("container-build") + temp_build_dir = container_parent_dir.joinpath(container.replace("/", "-")) + temp_build_script_filename = temp_build_dir.joinpath("build.sh") + # Now check if the container exists in the external stack. + if not temp_build_script_filename.exists(): + # If not, revert to building an internal container + container_parent_dir = container_build_dir + else: + container_parent_dir = container_build_dir + + build_dir = container_parent_dir.joinpath(container.replace("/", "-")) + build_script_filename = build_dir.joinpath("build.sh") + if verbose: print(f"Build script filename: {build_script_filename}") if os.path.exists(build_script_filename): - build_command = build_script_filename + build_command = build_script_filename.as_posix() else: if verbose: print(f"No script file found: {build_script_filename}, using default build script") @@ -84,7 +102,7 @@ def process_container(container, repo_full_path = os.path.join(dev_root_path, repo_dir) repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir build_command = os.path.join(container_build_dir, - "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}" + "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}" if not dry_run: if verbose: print(f"Executing: {build_command} with environment: {container_build_env}") @@ -158,7 +176,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args): for container in containers_in_scope: if include_exclude_check(container, include, exclude): - process_container(container, container_build_dir, container_build_env, + process_container(stack, container, container_build_dir, container_build_env, dev_root_path, quiet, verbose, dry_run, continue_on_error) else: if verbose: diff --git a/stack_orchestrator/build/build_webapp.py b/stack_orchestrator/build/build_webapp.py index 7a656ae7..ace334c4 100644 --- a/stack_orchestrator/build/build_webapp.py +++ b/stack_orchestrator/build/build_webapp.py @@ -60,7 +60,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args): container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug, force_rebuild, extra_build_args) - build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet, + build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet, verbose, dry_run, continue_on_error) @@ -73,5 +73,5 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args): webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1] container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local" - build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet, + build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet, verbose, dry_run, continue_on_error) diff --git a/stack_orchestrator/constants.py b/stack_orchestrator/constants.py new file mode 100644 index 00000000..1e24794c --- /dev/null +++ b/stack_orchestrator/constants.py @@ -0,0 +1,16 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +stack_file_name = "stack.yml" diff --git a/stack_orchestrator/repos/setup_repositories.py b/stack_orchestrator/repos/setup_repositories.py index feca7897..3612aed0 100644 --- a/stack_orchestrator/repos/setup_repositories.py +++ b/stack_orchestrator/repos/setup_repositories.py @@ -25,7 +25,8 @@ import click import importlib.resources from pathlib import Path import yaml -from stack_orchestrator.util import include_exclude_check +from stack_orchestrator.constants import stack_file_name +from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit class GitProgress(git.RemoteProgress): @@ -238,13 +239,20 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches repos_in_scope = [] if stack: - # In order to be compatible with Python 3.8 we need to use this hack to get the path: - # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure - stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, "stack.yml") + if stack_is_external(stack): + stack_file_path = Path(stack).joinpath(stack_file_name) + else: + # In order to be compatible with Python 3.8 we need to use this hack to get the path: + # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure + stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name) + if not stack_file_path.exists(): + error_exit(f"stack {stack} does not exist") with stack_file_path: stack_config = yaml.safe_load(open(stack_file_path, "r")) - # TODO: syntax check the input here - repos_in_scope = stack_config['repos'] + if "repos" not in stack_config: + error_exit(f"stack {stack} does not define any repositories") + else: + repos_in_scope = stack_config["repos"] else: repos_in_scope = all_repos diff --git a/stack_orchestrator/util.py b/stack_orchestrator/util.py index d3b733a2..97d48963 100644 --- a/stack_orchestrator/util.py +++ b/stack_orchestrator/util.py @@ -150,6 +150,12 @@ def get_parsed_deployment_spec(spec_file): sys.exit(1) +def stack_is_external(stack: str): + # Bit of a hack: if the supplied stack string represents + # a path that exists then we assume it must be external + return Path(stack).exists() if stack is not None else False + + def get_yaml(): # See: https://stackoverflow.com/a/45701840/1701505 yaml = ruamel.yaml.YAML() @@ -167,3 +173,8 @@ def global_options(ctx): # TODO: hack def global_options2(ctx): return ctx.parent.obj + + +def error_exit(s): + print(f"ERROR: {s}") + sys.exit(1)