Support external stack file (#650)

stack_orchestrator/build/build_containers.py

@@ -27,7 +27,7 @@ import subprocess
 import click
 import importlib.resources
 from pathlib import Path
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config
+from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external
 from stack_orchestrator.base import get_npm_registry_url
 
 # TODO: find a place for this
@@ -58,7 +58,8 @@ def make_container_build_env(dev_root_path: str,
     return container_build_env
 
 
-def process_container(container,
+def process_container(stack: str,
+                      container,
                       container_build_dir: str,
                       container_build_env: dict,
                       dev_root_path: str,
@@ -69,12 +70,29 @@ def process_container(container,
                       ):
     if not quiet:
         print(f"Building: {container}")
-    build_dir = os.path.join(container_build_dir, container.replace("/", "-"))
+
-    build_script_filename = os.path.join(build_dir, "build.sh")
+    default_container_tag = f"{container}:local"
+    container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
+
+    # Check if this is in an external stack
+    if stack_is_external(stack):
+        container_parent_dir = Path(stack).joinpath("container-build")
+        temp_build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
+        temp_build_script_filename = temp_build_dir.joinpath("build.sh")
+        # Now check if the container exists in the external stack.
+        if not temp_build_script_filename.exists():
+            # If not, revert to building an internal container
+            container_parent_dir = container_build_dir
+    else:
+        container_parent_dir = container_build_dir
+
+    build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
+    build_script_filename = build_dir.joinpath("build.sh")
+
     if verbose:
         print(f"Build script filename: {build_script_filename}")
     if os.path.exists(build_script_filename):
-        build_command = build_script_filename
+        build_command = build_script_filename.as_posix()
     else:
         if verbose:
             print(f"No script file found: {build_script_filename}, using default build script")
@@ -84,7 +102,7 @@ def process_container(container,
         repo_full_path = os.path.join(dev_root_path, repo_dir)
         repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
         build_command = os.path.join(container_build_dir,
-                                     "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}"
+                                     "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
     if not dry_run:
         if verbose:
             print(f"Executing: {build_command} with environment: {container_build_env}")
@@ -158,7 +176,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
 
     for container in containers_in_scope:
         if include_exclude_check(container, include, exclude):
-            process_container(container, container_build_dir, container_build_env,
+            process_container(stack, container, container_build_dir, container_build_env,
                               dev_root_path, quiet, verbose, dry_run, continue_on_error)
         else:
             if verbose:

stack_orchestrator/build/build_webapp.py

@@ -60,7 +60,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
     container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
                                                                     force_rebuild, extra_build_args)
 
-    build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet,
+    build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
                                        verbose, dry_run, continue_on_error)
 
 
@@ -73,5 +73,5 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
     webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
     container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
 
-    build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet,
+    build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
                                        verbose, dry_run, continue_on_error)

stack_orchestrator/constants.py

@@ -0,0 +1,16 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+stack_file_name = "stack.yml"

stack_orchestrator/repos/setup_repositories.py

@@ -25,7 +25,8 @@ import click
 import importlib.resources
 from pathlib import Path
 import yaml
-from stack_orchestrator.util import include_exclude_check
+from stack_orchestrator.constants import stack_file_name
+from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
 
 
 class GitProgress(git.RemoteProgress):
@@ -238,13 +239,20 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches
 
     repos_in_scope = []
     if stack:
-        # In order to be compatible with Python 3.8 we need to use this hack to get the path:
+        if stack_is_external(stack):
-        # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
+            stack_file_path = Path(stack).joinpath(stack_file_name)
-        stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, "stack.yml")
+        else:
+            # In order to be compatible with Python 3.8 we need to use this hack to get the path:
+            # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
+            stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name)
+        if not stack_file_path.exists():
+            error_exit(f"stack {stack} does not exist")
         with stack_file_path:
             stack_config = yaml.safe_load(open(stack_file_path, "r"))
-            # TODO: syntax check the input here
+            if "repos" not in stack_config:
-            repos_in_scope = stack_config['repos']
+                error_exit(f"stack {stack} does not define any repositories")
+            else:
+                repos_in_scope = stack_config["repos"]
     else:
         repos_in_scope = all_repos
 

stack_orchestrator/util.py

@@ -150,6 +150,12 @@ def get_parsed_deployment_spec(spec_file):
         sys.exit(1)
 
 
+def stack_is_external(stack: str):
+    # Bit of a hack: if the supplied stack string represents
+    # a path that exists then we assume it must be external
+    return Path(stack).exists() if stack is not None else False
+
+
 def get_yaml():
     # See: https://stackoverflow.com/a/45701840/1701505
     yaml = ruamel.yaml.YAML()
@@ -167,3 +173,8 @@ def global_options(ctx):
 # TODO: hack
 def global_options2(ctx):
     return ctx.parent.obj
+
+
+def error_exit(s):
+    print(f"ERROR: {s}")
+    sys.exit(1)