From ff69670db66fa19943653ae4efabcc0c99a96ab4 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 17 Feb 2023 13:34:51 -0700 Subject: [PATCH] Initial commit Former-commit-id: 60c1da725e0900045012a80d2449326d5c55fee7 --- app/base.py | 34 +++++++++++++++ app/build_npms.py | 10 ++++- app/data/stacks/package-registry/stack.yml | 11 +++++ app/deploy_system.py | 51 ++++++++++++++++++++-- 4 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 app/base.py create mode 100644 app/data/stacks/package-registry/stack.yml diff --git a/app/base.py b/app/base.py new file mode 100644 index 00000000..04cdaf5a --- /dev/null +++ b/app/base.py @@ -0,0 +1,34 @@ +# Copyright © 2022, 2023 Cerc + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +def get_stack(config, stack): + return base_stack(config, stack) + + +class base_stack(): + + def __init__(self, config, stack): + self.config = config + self.stack = stack + + def ensure_available(self): + if self.config.verbose: + print(f"Checking that base stack {self.stack} is available") + return 1 + + def get_url(self): + return "http://gitea.local:3000/api/packages/cerc-io/npm/" + +# need code to \ No newline at end of file diff --git a/app/build_npms.py b/app/build_npms.py index 3cf7d404..d794d027 100644 --- a/app/build_npms.py +++ b/app/build_npms.py @@ -24,6 +24,7 @@ from decouple import config import click import importlib.resources from python_on_whales import docker, DockerException +from .base import get_stack from .util import include_exclude_check, get_parsed_stack_config @click.command() @@ -41,6 +42,12 @@ def command(ctx, include, exclude): stack = ctx.obj.stack continue_on_error = ctx.obj.continue_on_error + # build-npms depends on having access to a writable package registry + # so we check here that it is available + package_registry_stack = get_stack(ctx.obj, 'package-registry') + package_registry_stack.ensure_available() + npm_registry_url = package_registry_stack.get_url('package-registry') + if local_stack: dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') @@ -75,7 +82,7 @@ def command(ctx, include, exclude): repo_dir = package repo_full_path = os.path.join(dev_root_path, repo_dir) # TODO: make the npm registry url configurable. - build_command = ["sh", "-c", "cd /workspace && build-npm-package-local-dependencies.sh http://gitea.local:3000/api/packages/cerc-io/npm/"] + build_command = ["sh", "-c", f"cd /workspace && build-npm-package-local-dependencies.sh {npm_registry_url}"] if not dry_run: if verbose: print(f"Executing: {build_command}") @@ -87,6 +94,7 @@ def command(ctx, include, exclude): tty=True, user=f"{os.getuid()}:{os.getgid()}", envs=envs, + # TODO: detect this host name in npm_registry_url rather than hard-wiring it add_hosts=[("gitea.local", "host-gateway")], volumes=[(repo_full_path, "/workspace")], command=build_command diff --git a/app/data/stacks/package-registry/stack.yml b/app/data/stacks/package-registry/stack.yml new file mode 100644 index 00000000..771149ed --- /dev/null +++ b/app/data/stacks/package-registry/stack.yml @@ -0,0 +1,11 @@ +version: "1.1" +name: package-registry +decription: "Local Package Registry" +repos: + - cerc-io/hosting +pods: + - name: gitea + repository: cerc-io/hosting + path: gitea + pre_start_command: "run-this-first.sh" + post_start_command: "initialize-gitea.sh" diff --git a/app/deploy_system.py b/app/deploy_system.py index b7fc69a6..f7bf71b1 100644 --- a/app/deploy_system.py +++ b/app/deploy_system.py @@ -18,6 +18,7 @@ import hashlib import os import sys +from decouple import config from python_on_whales import DockerClient import click import importlib.resources @@ -40,9 +41,16 @@ def command(ctx, include, exclude, cluster, command, extra_args): debug = ctx.obj.debug quiet = ctx.obj.quiet verbose = ctx.obj.verbose + local_stack = ctx.obj.local_stack dry_run = ctx.obj.dry_run stack = ctx.obj.stack + if local_stack: + dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] + print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') + else: + dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) + # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure compose_dir = Path(__file__).absolute().parent.joinpath("data", "compose") @@ -68,19 +76,37 @@ def command(ctx, include, exclude, cluster, command, extra_args): else: pods_in_scope = all_pods + # Convert all pod definitions to v1.1 format + pods_in_scope = _convert_to_new_format(pods_in_scope) + if verbose: print(f"Pods: {pods_in_scope}") # Construct a docker compose command suitable for our purpose compose_files = [] + pre_start_commands = [] + post_start_commands = [] for pod in pods_in_scope: - if include_exclude_check(pod, include, exclude): - compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod}.yml") + pod_name = pod["name"] + pod_repository = pod["repository"] + pod_path = pod["path"] + if include_exclude_check(pod_name, include, exclude): + if pod_repository is None or pod_repository == "internal": + compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml") + else: + pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"]) + compose_file_name = os.path.join(pod_root_dir, "docker-compose.yml") + pod_pre_start_command = pod["pre_start_command"] + pod_post_start_command = pod["post_start_command"] + if pod_pre_start_command is not None: + pre_start_commands.append(os.path.join(pod_root_dir, pod_pre_start_command)) + if pod_post_start_command is not None: + post_start_commands.append(os.path.join(pod_root_dir, pod_post_start_command)) compose_files.append(compose_file_name) else: if verbose: - print(f"Excluding: {pod}") + print(f"Excluding: {pod_name}") if verbose: print(f"files: {compose_files}") @@ -96,7 +122,11 @@ def command(ctx, include, exclude, cluster, command, extra_args): os.environ["CERC_SCRIPT_DEBUG"] = "true" if verbose: print(f"Running compose up for extra_args: {extra_args_list}") + for pre_start_command in pre_start_commands: + 1 docker.compose.up(detach=True, services=extra_args_list) + for post_start_command in post_start_commands: + 1 elif command == "down": if verbose: print("Running compose down") @@ -148,3 +178,18 @@ def command(ctx, include, exclude, cluster, command, extra_args): if verbose: print("Running compose logs") docker.compose.logs() + + +def _convert_to_new_format(old_pod_array): + new_pod_array = [] + for old_pod in old_pod_array: + if isinstance(old_pod, dict): + new_pod_array.append(old_pod) + else: + new_pod = { + "name": old_pod, + "repository": "internal", + "path": old_pod + } + new_pod_array.append(new_pod) + return new_pod_array