From 63ce394892387bc733e5dfbcb2486bfd9ece39b9 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 14 Nov 2023 18:53:29 -0700 Subject: [PATCH 1/7] Check stack is a path --- stack_orchestrator/constants.py | 16 ++++++++++++++++ stack_orchestrator/repos/setup_repositories.py | 10 +++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 stack_orchestrator/constants.py diff --git a/stack_orchestrator/constants.py b/stack_orchestrator/constants.py new file mode 100644 index 00000000..94d92e73 --- /dev/null +++ b/stack_orchestrator/constants.py @@ -0,0 +1,16 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +stack_file_name = "stack.yml" \ No newline at end of file diff --git a/stack_orchestrator/repos/setup_repositories.py b/stack_orchestrator/repos/setup_repositories.py index feca7897..3f817ec2 100644 --- a/stack_orchestrator/repos/setup_repositories.py +++ b/stack_orchestrator/repos/setup_repositories.py @@ -25,6 +25,7 @@ import click import importlib.resources from pathlib import Path import yaml +from stack_orchestrator.constants import stack_file_name from stack_orchestrator.util import include_exclude_check @@ -238,9 +239,12 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches repos_in_scope = [] if stack: - # In order to be compatible with Python 3.8 we need to use this hack to get the path: - # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure - stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, "stack.yml") + # Check if the stack argument is a path or a plain string. + stack_file_path = Path(stack).joinpath(stack_file_name) + if not stack_file_path.exists(): + # In order to be compatible with Python 3.8 we need to use this hack to get the path: + # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure + stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name) with stack_file_path: stack_config = yaml.safe_load(open(stack_file_path, "r")) # TODO: syntax check the input here -- 2.45.2 From 918ef25f1a9f40eaaae90972e14610770fad5a49 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 14 Nov 2023 18:59:54 -0700 Subject: [PATCH 2/7] Fail gracefully on bad stack --- stack_orchestrator/constants.py | 2 +- stack_orchestrator/repos/setup_repositories.py | 4 +++- stack_orchestrator/util.py | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/stack_orchestrator/constants.py b/stack_orchestrator/constants.py index 94d92e73..1e24794c 100644 --- a/stack_orchestrator/constants.py +++ b/stack_orchestrator/constants.py @@ -13,4 +13,4 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -stack_file_name = "stack.yml" \ No newline at end of file +stack_file_name = "stack.yml" diff --git a/stack_orchestrator/repos/setup_repositories.py b/stack_orchestrator/repos/setup_repositories.py index 3f817ec2..4e0f883d 100644 --- a/stack_orchestrator/repos/setup_repositories.py +++ b/stack_orchestrator/repos/setup_repositories.py @@ -26,7 +26,7 @@ import importlib.resources from pathlib import Path import yaml from stack_orchestrator.constants import stack_file_name -from stack_orchestrator.util import include_exclude_check +from stack_orchestrator.util import include_exclude_check, error_exit class GitProgress(git.RemoteProgress): @@ -245,6 +245,8 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches # In order to be compatible with Python 3.8 we need to use this hack to get the path: # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name) + if not stack_file_path.exists(): + error_exit(f"stack {stack} does not exist") with stack_file_path: stack_config = yaml.safe_load(open(stack_file_path, "r")) # TODO: syntax check the input here diff --git a/stack_orchestrator/util.py b/stack_orchestrator/util.py index d3b733a2..2d63991a 100644 --- a/stack_orchestrator/util.py +++ b/stack_orchestrator/util.py @@ -167,3 +167,8 @@ def global_options(ctx): # TODO: hack def global_options2(ctx): return ctx.parent.obj + + +def error_exit(s): + print(f"ERROR: {s}") + sys.exit(1) -- 2.45.2 From daf099b5eae87b1ca07f095e2a0944d8c356b01d Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 14 Nov 2023 20:41:11 -0700 Subject: [PATCH 3/7] Implement container build support --- stack_orchestrator/build/build_containers.py | 32 +++++++++++++++---- .../repos/setup_repositories.py | 8 ++--- stack_orchestrator/util.py | 6 ++++ 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/stack_orchestrator/build/build_containers.py b/stack_orchestrator/build/build_containers.py index 5b2748cc..e987c504 100644 --- a/stack_orchestrator/build/build_containers.py +++ b/stack_orchestrator/build/build_containers.py @@ -27,7 +27,7 @@ import subprocess import click import importlib.resources from pathlib import Path -from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config +from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external from stack_orchestrator.base import get_npm_registry_url # TODO: find a place for this @@ -58,7 +58,8 @@ def make_container_build_env(dev_root_path: str, return container_build_env -def process_container(container, +def process_container(stack: str, + container, container_build_dir: str, container_build_env: dict, dev_root_path: str, @@ -69,12 +70,29 @@ def process_container(container, ): if not quiet: print(f"Building: {container}") - build_dir = os.path.join(container_build_dir, container.replace("/", "-")) - build_script_filename = os.path.join(build_dir, "build.sh") + + default_container_tag = f"{container}:local" + container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag}) + + # Check if this is in an external stack + if stack_is_external(stack): + container_parent_dir = Path(stack).joinpath("container-build") + temp_build_dir = container_parent_dir.joinpath(container.replace("/", "-")) + temp_build_script_filename = temp_build_dir.joinpath("build.sh") + # Now check if the container exists in the external stack. + if not temp_build_script_filename.exists(): + # If not, revert to building an internal container + container_parent_dir = container_build_dir + else: + container_parent_dir = container_build_dir + + build_dir = container_parent_dir.joinpath(container.replace("/", "-")) + build_script_filename = build_dir.joinpath("build.sh") + if verbose: print(f"Build script filename: {build_script_filename}") if os.path.exists(build_script_filename): - build_command = build_script_filename + build_command = build_script_filename.as_posix() else: if verbose: print(f"No script file found: {build_script_filename}, using default build script") @@ -84,7 +102,7 @@ def process_container(container, repo_full_path = os.path.join(dev_root_path, repo_dir) repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir build_command = os.path.join(container_build_dir, - "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}" + "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}" if not dry_run: if verbose: print(f"Executing: {build_command} with environment: {container_build_env}") @@ -158,7 +176,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args): for container in containers_in_scope: if include_exclude_check(container, include, exclude): - process_container(container, container_build_dir, container_build_env, + process_container(stack, container, container_build_dir, container_build_env, dev_root_path, quiet, verbose, dry_run, continue_on_error) else: if verbose: diff --git a/stack_orchestrator/repos/setup_repositories.py b/stack_orchestrator/repos/setup_repositories.py index 4e0f883d..b3c80aeb 100644 --- a/stack_orchestrator/repos/setup_repositories.py +++ b/stack_orchestrator/repos/setup_repositories.py @@ -26,7 +26,7 @@ import importlib.resources from pathlib import Path import yaml from stack_orchestrator.constants import stack_file_name -from stack_orchestrator.util import include_exclude_check, error_exit +from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit class GitProgress(git.RemoteProgress): @@ -239,9 +239,9 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches repos_in_scope = [] if stack: - # Check if the stack argument is a path or a plain string. - stack_file_path = Path(stack).joinpath(stack_file_name) - if not stack_file_path.exists(): + if stack_is_external(stack): + stack_file_path = Path(stack).joinpath(stack_file_name) + else: # In order to be compatible with Python 3.8 we need to use this hack to get the path: # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name) diff --git a/stack_orchestrator/util.py b/stack_orchestrator/util.py index 2d63991a..98cfd611 100644 --- a/stack_orchestrator/util.py +++ b/stack_orchestrator/util.py @@ -150,6 +150,12 @@ def get_parsed_deployment_spec(spec_file): sys.exit(1) +def stack_is_external(stack: str): + # Bit of a hack: if the supplied stack string represents + # a path that exists then we assume it must be external + return Path(stack).exists() + + def get_yaml(): # See: https://stackoverflow.com/a/45701840/1701505 yaml = ruamel.yaml.YAML() -- 2.45.2 From 22b88b438ffb5086401ef43b46f94fdfcf433c09 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 14 Nov 2023 20:46:49 -0700 Subject: [PATCH 4/7] Add error checking --- stack_orchestrator/repos/setup_repositories.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/stack_orchestrator/repos/setup_repositories.py b/stack_orchestrator/repos/setup_repositories.py index b3c80aeb..29f05bb0 100644 --- a/stack_orchestrator/repos/setup_repositories.py +++ b/stack_orchestrator/repos/setup_repositories.py @@ -249,8 +249,10 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches error_exit(f"stack {stack} does not exist") with stack_file_path: stack_config = yaml.safe_load(open(stack_file_path, "r")) - # TODO: syntax check the input here - repos_in_scope = stack_config['repos'] + if not "repos" in stack_config: + error_exit(f"stack {stack} does not define any repositories") + else: + repos_in_scope = stack_config["repos"] else: repos_in_scope = all_repos -- 2.45.2 From 9cb10d200d97245e76d06dc8428b87f139342984 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 14 Nov 2023 20:49:38 -0700 Subject: [PATCH 5/7] Fix lint error --- stack_orchestrator/repos/setup_repositories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack_orchestrator/repos/setup_repositories.py b/stack_orchestrator/repos/setup_repositories.py index 29f05bb0..3612aed0 100644 --- a/stack_orchestrator/repos/setup_repositories.py +++ b/stack_orchestrator/repos/setup_repositories.py @@ -249,7 +249,7 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches error_exit(f"stack {stack} does not exist") with stack_file_path: stack_config = yaml.safe_load(open(stack_file_path, "r")) - if not "repos" in stack_config: + if "repos" not in stack_config: error_exit(f"stack {stack} does not define any repositories") else: repos_in_scope = stack_config["repos"] -- 2.45.2 From 7d7527d7773387b1929086e7a3973501c420af0c Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 14 Nov 2023 20:54:22 -0700 Subject: [PATCH 6/7] Fix no stack case --- stack_orchestrator/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack_orchestrator/util.py b/stack_orchestrator/util.py index 98cfd611..97d48963 100644 --- a/stack_orchestrator/util.py +++ b/stack_orchestrator/util.py @@ -153,7 +153,7 @@ def get_parsed_deployment_spec(spec_file): def stack_is_external(stack: str): # Bit of a hack: if the supplied stack string represents # a path that exists then we assume it must be external - return Path(stack).exists() + return Path(stack).exists() if stack is not None else False def get_yaml(): -- 2.45.2 From dba033dd2b8b872b99b3e63b402e5297abed1a6a Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 14 Nov 2023 20:56:37 -0700 Subject: [PATCH 7/7] Fix build-webapp command --- stack_orchestrator/build/build_webapp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stack_orchestrator/build/build_webapp.py b/stack_orchestrator/build/build_webapp.py index 7a656ae7..ace334c4 100644 --- a/stack_orchestrator/build/build_webapp.py +++ b/stack_orchestrator/build/build_webapp.py @@ -60,7 +60,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args): container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug, force_rebuild, extra_build_args) - build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet, + build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet, verbose, dry_run, continue_on_error) @@ -73,5 +73,5 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args): webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1] container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local" - build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet, + build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet, verbose, dry_run, continue_on_error) -- 2.45.2