Merge pull request #178 from cerc-io/dboreham/package-registry-stack

Support for the package registry stack

Former-commit-id: e5197e4918b4b63a19c2d675f8f02a1a00c5e676

app/base.py

@@ -0,0 +1,34 @@
+# Copyright © 2022, 2023 Cerc
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+def get_stack(config, stack):
+    return base_stack(config, stack)
+
+
+class base_stack():
+
+    def __init__(self, config, stack):
+        self.config = config
+        self.stack = stack
+
+    def ensure_available(self):
+        if self.config.verbose:
+            print(f"Checking that base stack {self.stack} is available")
+        return 1
+
+    def get_url(self):
+        return "http://gitea.local:3000/api/packages/cerc-io/npm/"
+
+# TODO: finish this implementation for the npm package registry

app/build_npms.py

@@ -24,6 +24,7 @@ from decouple import config
 import click
 import importlib.resources
 from python_on_whales import docker, DockerException
+from .base import get_stack
 from .util import include_exclude_check, get_parsed_stack_config
 
 @click.command()
@@ -41,6 +42,12 @@ def command(ctx, include, exclude):
     stack = ctx.obj.stack
     continue_on_error = ctx.obj.continue_on_error
 
+    # build-npms depends on having access to a writable package registry
+    # so we check here that it is available
+    package_registry_stack = get_stack(ctx.obj, 'package-registry')
+    package_registry_stack.ensure_available()
+    npm_registry_url = package_registry_stack.get_url('package-registry')
+
     if local_stack:
         dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
         print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
@@ -75,7 +82,7 @@ def command(ctx, include, exclude):
         repo_dir = package
         repo_full_path = os.path.join(dev_root_path, repo_dir)
         # TODO: make the npm registry url configurable.
-        build_command = ["sh", "-c", "cd /workspace && build-npm-package-local-dependencies.sh http://gitea.local:3000/api/packages/cerc-io/npm/"]
+        build_command = ["sh", "-c", f"cd /workspace && build-npm-package-local-dependencies.sh {npm_registry_url}"]
         if not dry_run:
             if verbose:
                 print(f"Executing: {build_command}")
@@ -87,6 +94,7 @@ def command(ctx, include, exclude):
                            tty=True,
                            user=f"{os.getuid()}:{os.getgid()}",
                            envs=envs,
+                           # TODO: detect this host name in npm_registry_url rather than hard-wiring it
                            add_hosts=[("gitea.local", "host-gateway")],
                            volumes=[(repo_full_path, "/workspace")],
                            command=build_command

app/data/stacks/package-registry/stack.yml

@@ -0,0 +1,11 @@
+version: "1.1"
+name: package-registry
+decription: "Local Package Registry"
+repos:
+  - cerc-io/hosting
+pods:
+  - name: gitea
+    repository: cerc-io/hosting
+    path: gitea
+    pre_start_command: "run-this-first.sh"
+    post_start_command: "initialize-gitea.sh"

app/deploy_system.py

@@ -18,6 +18,8 @@
 import hashlib
 import os
 import sys
+from decouple import config
+import subprocess
 from python_on_whales import DockerClient
 import click
 import importlib.resources
@@ -40,9 +42,16 @@ def command(ctx, include, exclude, cluster, command, extra_args):
     debug = ctx.obj.debug
     quiet = ctx.obj.quiet
     verbose = ctx.obj.verbose
+    local_stack = ctx.obj.local_stack
     dry_run = ctx.obj.dry_run
     stack = ctx.obj.stack
 
+    if local_stack:
+        dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
+        print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
+    else:
+        dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
+
     # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
     compose_dir = Path(__file__).absolute().parent.joinpath("data", "compose")
 
@@ -68,19 +77,37 @@ def command(ctx, include, exclude, cluster, command, extra_args):
     else:
         pods_in_scope = all_pods
 
+    # Convert all pod definitions to v1.1 format
+    pods_in_scope = _convert_to_new_format(pods_in_scope)
+
     if verbose:
         print(f"Pods: {pods_in_scope}")
 
     # Construct a docker compose command suitable for our purpose
 
     compose_files = []
+    pre_start_commands = []
+    post_start_commands = []
     for pod in pods_in_scope:
-        if include_exclude_check(pod, include, exclude):
+        pod_name = pod["name"]
-            compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod}.yml")
+        pod_repository = pod["repository"]
+        pod_path = pod["path"]
+        if include_exclude_check(pod_name, include, exclude):
+            if pod_repository is None or pod_repository == "internal":
+                compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml")
+            else:
+                pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
+                compose_file_name = os.path.join(pod_root_dir, "docker-compose.yml")
+                pod_pre_start_command = pod["pre_start_command"]
+                pod_post_start_command = pod["post_start_command"]
+                if pod_pre_start_command is not None:
+                    pre_start_commands.append(os.path.join(pod_root_dir, pod_pre_start_command))
+                if pod_post_start_command is not None:
+                    post_start_commands.append(os.path.join(pod_root_dir, pod_post_start_command))
             compose_files.append(compose_file_name)
         else:
             if verbose:
-                print(f"Excluding: {pod}")
+                print(f"Excluding: {pod_name}")
 
     if verbose:
         print(f"files: {compose_files}")
@@ -96,7 +123,11 @@ def command(ctx, include, exclude, cluster, command, extra_args):
                 os.environ["CERC_SCRIPT_DEBUG"] = "true"
             if verbose:
                 print(f"Running compose up for extra_args: {extra_args_list}")
+            for pre_start_command in pre_start_commands:
+                _run_command(ctx.obj, cluster, pre_start_command)
             docker.compose.up(detach=True, services=extra_args_list)
+            for post_start_command in post_start_commands:
+                _run_command(ctx.obj, cluster, post_start_command)
         elif command == "down":
             if verbose:
                 print("Running compose down")
@@ -148,3 +179,34 @@ def command(ctx, include, exclude, cluster, command, extra_args):
             if verbose:
                 print("Running compose logs")
             docker.compose.logs()
+
+
+def _convert_to_new_format(old_pod_array):
+    new_pod_array = []
+    for old_pod in old_pod_array:
+        if isinstance(old_pod, dict):
+            new_pod_array.append(old_pod)
+        else:
+            new_pod = {
+                "name": old_pod,
+                "repository": "internal",
+                "path": old_pod
+            }
+            new_pod_array.append(new_pod)
+    return new_pod_array
+
+
+def _run_command(ctx, cluster_name, command):
+    if ctx.verbose:
+        print(f"Running command: {command}")
+    command_dir = os.path.dirname(command)
+    print(f"command_dir: {command_dir}")
+    command_file = os.path.join(".", os.path.basename(command))
+    command_env = os.environ.copy()
+    command_env["CERC_SO_COMPOSE_PROJECT"] = cluster_name
+    if ctx.debug:
+        command_env["CERC_SCRIPT_DEBUG"] = "true"
+    command_result = subprocess.run(command_file, shell=True, env=command_env, cwd=command_dir)
+    if command_result.returncode != 0:
+        print(f"FATAL Error running command: {command}")
+        sys.exit(1)