roysc/stack-orchestrator
1
0
Fork 0
Code Pull Requests Activity

Merge pull request #178 from cerc-io/dboreham/package-registry-stack

Browse Source 
Support for the package registry stack

Former-commit-id: e5197e4918b4b63a19c2d675f8f02a1a00c5e676
This commit is contained in:
David Boreham 2023-02-17 15:36:28 -07:00 committed by GitHub
commit e20f9993d2
4 changed files with 119 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
app/base.py Normal file
View File

@ -0,0 +1,34 @@
# Copyright © 2022, 2023 Cerc
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
def get_stack(config, stack):
return base_stack(config, stack)
class base_stack():
def __init__(self, config, stack):
self.config = config
self.stack = stack
def ensure_available(self):
if self.config.verbose:
print(f"Checking that base stack {self.stack} is available")
return 1
def get_url(self):
return "http://gitea.local:3000/api/packages/cerc-io/npm/"
# TODO: finish this implementation for the npm package registry

10
app/build_npms.py
View File

@ -24,6 +24,7 @@ from decouple import config
import click
import importlib.resources
from python_on_whales import docker, DockerException
from .base import get_stack
from .util import include_exclude_check, get_parsed_stack_config
@click.command()
@ -41,6 +42,12 @@ def command(ctx, include, exclude):
stack = ctx.obj.stack
continue_on_error = ctx.obj.continue_on_error
# build-npms depends on having access to a writable package registry
# so we check here that it is available
package_registry_stack = get_stack(ctx.obj, 'package-registry')
package_registry_stack.ensure_available()
npm_registry_url = package_registry_stack.get_url('package-registry')
if local_stack:
dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
@ -75,7 +82,7 @@ def command(ctx, include, exclude):
repo_dir = package
repo_full_path = os.path.join(dev_root_path, repo_dir)
# TODO: make the npm registry url configurable.
build_command = ["sh", "-c", "cd /workspace && build-npm-package-local-dependencies.sh http://gitea.local:3000/api/packages/cerc-io/npm/"]
build_command = ["sh", "-c", f"cd /workspace && build-npm-package-local-dependencies.sh {npm_registry_url}"]
if not dry_run:
if verbose:
print(f"Executing: {build_command}")
@ -87,6 +94,7 @@ def command(ctx, include, exclude):
tty=True,
user=f"{os.getuid()}:{os.getgid()}",
envs=envs,
# TODO: detect this host name in npm_registry_url rather than hard-wiring it
add_hosts=[("gitea.local", "host-gateway")],
volumes=[(repo_full_path, "/workspace")],
command=build_command

11
app/data/stacks/package-registry/stack.yml Normal file
View File

@ -0,0 +1,11 @@
version: "1.1"
name: package-registry
decription: "Local Package Registry"
repos:
- cerc-io/hosting
pods:
- name: gitea
repository: cerc-io/hosting
path: gitea
pre_start_command: "run-this-first.sh"
post_start_command: "initialize-gitea.sh"

68
app/deploy_system.py
View File

@ -18,6 +18,8 @@
import hashlib
import os
import sys
from decouple import config
import subprocess
from python_on_whales import DockerClient
import click
import importlib.resources
@ -40,9 +42,16 @@ def command(ctx, include, exclude, cluster, command, extra_args):
debug = ctx.obj.debug
quiet = ctx.obj.quiet
verbose = ctx.obj.verbose
local_stack = ctx.obj.local_stack
dry_run = ctx.obj.dry_run
stack = ctx.obj.stack
if local_stack:
dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
else:
dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
compose_dir = Path(__file__).absolute().parent.joinpath("data", "compose")
@ -68,19 +77,37 @@ def command(ctx, include, exclude, cluster, command, extra_args):
else:
pods_in_scope = all_pods
# Convert all pod definitions to v1.1 format
pods_in_scope = _convert_to_new_format(pods_in_scope)
if verbose:
print(f"Pods: {pods_in_scope}")
# Construct a docker compose command suitable for our purpose
compose_files = []
pre_start_commands = []
post_start_commands = []
for pod in pods_in_scope:
if include_exclude_check(pod, include, exclude):
compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod}.yml")
pod_name = pod["name"]
pod_repository = pod["repository"]
pod_path = pod["path"]
if include_exclude_check(pod_name, include, exclude):
if pod_repository is None or pod_repository == "internal":
compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml")
else:
pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
compose_file_name = os.path.join(pod_root_dir, "docker-compose.yml")
pod_pre_start_command = pod["pre_start_command"]
pod_post_start_command = pod["post_start_command"]
if pod_pre_start_command is not None:
pre_start_commands.append(os.path.join(pod_root_dir, pod_pre_start_command))
if pod_post_start_command is not None:
post_start_commands.append(os.path.join(pod_root_dir, pod_post_start_command))
compose_files.append(compose_file_name)
else:
if verbose:
print(f"Excluding: {pod}")
print(f"Excluding: {pod_name}")
if verbose:
print(f"files: {compose_files}")
@ -96,7 +123,11 @@ def command(ctx, include, exclude, cluster, command, extra_args):
os.environ["CERC_SCRIPT_DEBUG"] = "true"
if verbose:
print(f"Running compose up for extra_args: {extra_args_list}")
for pre_start_command in pre_start_commands:
_run_command(ctx.obj, cluster, pre_start_command)
docker.compose.up(detach=True, services=extra_args_list)
for post_start_command in post_start_commands:
_run_command(ctx.obj, cluster, post_start_command)
elif command == "down":
if verbose:
print("Running compose down")
@ -148,3 +179,34 @@ def command(ctx, include, exclude, cluster, command, extra_args):
if verbose:
print("Running compose logs")
docker.compose.logs()
def _convert_to_new_format(old_pod_array):
new_pod_array = []
for old_pod in old_pod_array:
if isinstance(old_pod, dict):
new_pod_array.append(old_pod)
else:
new_pod = {
"name": old_pod,
"repository": "internal",
"path": old_pod
}
new_pod_array.append(new_pod)
return new_pod_array
def _run_command(ctx, cluster_name, command):
if ctx.verbose:
print(f"Running command: {command}")
command_dir = os.path.dirname(command)
print(f"command_dir: {command_dir}")
command_file = os.path.join(".", os.path.basename(command))
command_env = os.environ.copy()
command_env["CERC_SO_COMPOSE_PROJECT"] = cluster_name
if ctx.debug:
command_env["CERC_SCRIPT_DEBUG"] = "true"
command_result = subprocess.run(command_file, shell=True, env=command_env, cwd=command_dir)
if command_result.returncode != 0:
print(f"FATAL Error running command: {command}")
sys.exit(1)