From d311fe73d0025e2c3324d38f37abadc3cd51b34b Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 26 Oct 2023 19:58:42 -0500 Subject: [PATCH 1/3] Split act-runner into its own pod and offer as a distinct stack. --- .../cerc-act-runner-task-executor/build.sh | 2 +- app/data/stacks/act-runner/README.md | 15 ++++ app/data/stacks/act-runner/stack.yml | 15 ++++ app/data/stacks/package-registry/stack.yml | 5 ++ app/deploy/deploy.py | 4 +- app/deploy/deployment_create.py | 72 ++++++++++--------- app/util.py | 16 ++--- 7 files changed, 85 insertions(+), 44 deletions(-) create mode 100644 app/data/stacks/act-runner/README.md create mode 100644 app/data/stacks/act-runner/stack.yml diff --git a/app/data/container-build/cerc-act-runner-task-executor/build.sh b/app/data/container-build/cerc-act-runner-task-executor/build.sh index 25620a53..b625ed4b 100755 --- a/app/data/container-build/cerc-act-runner-task-executor/build.sh +++ b/app/data/container-build/cerc-act-runner-task-executor/build.sh @@ -2,4 +2,4 @@ # Build a local version of the task executor for act-runner source ${CERC_CONTAINER_BASE_DIR}/build-base.sh SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/gitea/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR} +docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/act-runner/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR} diff --git a/app/data/stacks/act-runner/README.md b/app/data/stacks/act-runner/README.md new file mode 100644 index 00000000..e623a9c6 --- /dev/null +++ b/app/data/stacks/act-runner/README.md @@ -0,0 +1,15 @@ +# act-runner stack + +## Example + +``` +$ laconic-so --stack act-runner deploy init --output act-runner.yml + +$ laconic-so --stack act-runner deploy create --spec-file act-runner.yml --deployment-dir ~/opt/deployments/act-runner-1 +$ echo "CERC_GITEA_RUNNER_REGISTRATION_TOKEN=FOO" >> ~/opt/deployments/act-runner-1/config.env +$ laconic-so deployment --dir ~/opt/deployments/act-runner-1 up + +$ laconic-so --stack act-runner deploy create --spec-file act-runner.yml --deployment-dir ~/opt/deployments/act-runner-2 +$ echo "CERC_GITEA_RUNNER_REGISTRATION_TOKEN=BAR" >> ~/opt/deployments/act-runner-2/config.env +$ laconic-so deployment --dir ~/opt/deployments/act-runner-2 up +``` diff --git a/app/data/stacks/act-runner/stack.yml b/app/data/stacks/act-runner/stack.yml new file mode 100644 index 00000000..a236fccf --- /dev/null +++ b/app/data/stacks/act-runner/stack.yml @@ -0,0 +1,15 @@ +version: "1.1" +name: act-runner +description: "Local act-runner" +repos: + - git.vdb.to/cerc-io/hosting + - gitea.com/gitea/act_runner +containers: + - cerc/act-runner + - cerc/act-runner-task-executor +pods: + - name: act-runner + repository: cerc-io/hosting + path: act-runner + pre_start_command: "pre_start.sh" + post_start_command: "post_start.sh" diff --git a/app/data/stacks/package-registry/stack.yml b/app/data/stacks/package-registry/stack.yml index 33c6c939..f6367ab1 100644 --- a/app/data/stacks/package-registry/stack.yml +++ b/app/data/stacks/package-registry/stack.yml @@ -13,3 +13,8 @@ pods: path: gitea pre_start_command: "run-this-first.sh" post_start_command: "initialize-gitea.sh" + - name: act-runner + repository: cerc-io/hosting + path: act-runner + pre_start_command: "pre_start.sh" + post_start_command: "post_start.sh" diff --git a/app/deploy/deploy.py b/app/deploy/deploy.py index 40cd0a8d..a2d7cc01 100644 --- a/app/deploy/deploy.py +++ b/app/deploy/deploy.py @@ -307,7 +307,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml") else: if deployment: - compose_file_name = os.path.join(compose_dir, "docker-compose.yml") + compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml") pod_pre_start_command = pod["pre_start_command"] pod_post_start_command = pod["post_start_command"] script_dir = compose_dir.parent.joinpath("pods", pod_name, "scripts") @@ -317,7 +317,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): post_start_commands.append(os.path.join(script_dir, pod_post_start_command)) else: pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"]) - compose_file_name = os.path.join(pod_root_dir, "docker-compose.yml") + compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml") pod_pre_start_command = pod["pre_start_command"] pod_post_start_command = pod["post_start_command"] if pod_pre_start_command is not None: diff --git a/app/deploy/deployment_create.py b/app/deploy/deployment_create.py index dcaccb2b..b32d95f4 100644 --- a/app/deploy/deployment_create.py +++ b/app/deploy/deployment_create.py @@ -22,7 +22,7 @@ import random from shutil import copy, copyfile, copytree import sys from app.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml, - get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_path) + get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_paths) from app.deploy.deploy_types import DeploymentContext, DeployCommandContext, LaconicStackSetupCommand @@ -106,9 +106,10 @@ def _fixup_pod_file(pod, spec, compose_dir): pod["services"][container_name]["ports"] = container_ports -def _commands_plugin_path(ctx: DeployCommandContext): - plugin_path = get_plugin_code_path(ctx.stack) - return plugin_path.joinpath("deploy", "commands.py") +def _commands_plugin_paths(ctx: DeployCommandContext): + plugin_paths = get_plugin_code_paths(ctx.stack) + ret = [p.joinpath("deploy", "commands.py") for p in plugin_paths] + return ret # See: https://stackoverflow.com/a/54625079/1701505 @@ -120,15 +121,23 @@ def call_stack_deploy_init(deploy_command_context): # Link with the python file in the stack # Call a function in it # If no function found, return None - python_file_path = _commands_plugin_path(deploy_command_context) - if python_file_path.exists(): - spec = util.spec_from_file_location("commands", python_file_path) - imported_stack = util.module_from_spec(spec) - spec.loader.exec_module(imported_stack) - if _has_method(imported_stack, "init"): - return imported_stack.init(deploy_command_context) - else: - return None + python_file_paths = _commands_plugin_paths(deploy_command_context) + + ret = None + init_done = False + for python_file_path in python_file_paths: + if python_file_path.exists(): + spec = util.spec_from_file_location("commands", python_file_path) + imported_stack = util.module_from_spec(spec) + spec.loader.exec_module(imported_stack) + if _has_method(imported_stack, "init"): + if not init_done: + ret = imported_stack.init(deploy_command_context) + init_done = True + else: + #TODO: remove this restriction + print(f"Skipping init() from plugin {python_file_path}. Only one init() is allowed.") + return ret # TODO: fold this with function above @@ -136,16 +145,14 @@ def call_stack_deploy_setup(deploy_command_context, parameters: LaconicStackSetu # Link with the python file in the stack # Call a function in it # If no function found, return None - python_file_path = _commands_plugin_path(deploy_command_context) - print(f"Path: {python_file_path}") - if python_file_path.exists(): - spec = util.spec_from_file_location("commands", python_file_path) - imported_stack = util.module_from_spec(spec) - spec.loader.exec_module(imported_stack) - if _has_method(imported_stack, "setup"): - return imported_stack.setup(deploy_command_context, parameters, extra_args) - else: - return None + python_file_paths = _commands_plugin_paths(deploy_command_context) + for python_file_path in python_file_paths: + if python_file_path.exists(): + spec = util.spec_from_file_location("commands", python_file_path) + imported_stack = util.module_from_spec(spec) + spec.loader.exec_module(imported_stack) + if _has_method(imported_stack, "setup"): + imported_stack.setup(deploy_command_context, parameters, extra_args) # TODO: fold this with function above @@ -153,15 +160,14 @@ def call_stack_deploy_create(deployment_context, extra_args): # Link with the python file in the stack # Call a function in it # If no function found, return None - python_file_path = _commands_plugin_path(deployment_context.command_context) - if python_file_path.exists(): - spec = util.spec_from_file_location("commands", python_file_path) - imported_stack = util.module_from_spec(spec) - spec.loader.exec_module(imported_stack) - if _has_method(imported_stack, "create"): - return imported_stack.create(deployment_context, extra_args) - else: - return None + python_file_paths = _commands_plugin_paths(deployment_context.command_context) + for python_file_path in python_file_paths: + if python_file_path.exists(): + spec = util.spec_from_file_location("commands", python_file_path) + imported_stack = util.module_from_spec(spec) + spec.loader.exec_module(imported_stack) + if _has_method(imported_stack, "create"): + imported_stack.create(deployment_context, extra_args) # Inspect the pod yaml to find config files referenced in subdirectories @@ -336,7 +342,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): if global_options(ctx).debug: print(f"extra config dirs: {extra_config_dirs}") _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir) - with open(os.path.join(destination_compose_dir, os.path.basename(pod_file_path)), "w") as output_file: + with open(os.path.join(destination_compose_dir, "docker-compose-%s.yml" % pod), "w") as output_file: yaml.dump(parsed_pod_file, output_file) # Copy the config files for the pod, if any config_dirs = {pod} diff --git a/app/util.py b/app/util.py index a25aacdb..8aa751c5 100644 --- a/app/util.py +++ b/app/util.py @@ -79,16 +79,16 @@ def get_pod_list(parsed_stack): return result -def get_plugin_code_path(stack): +def get_plugin_code_paths(stack): parsed_stack = get_parsed_stack_config(stack) pods = parsed_stack["pods"] - # TODO: Hack - pod = pods[0] - if type(pod) is str: - result = get_stack_file_path(stack).parent - else: - pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"]) - result = Path(os.path.join(pod_root_dir, "stack")) + result = [] + for pod in pods: + if type(pod) is str: + result.append(get_stack_file_path(stack).parent) + else: + pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"]) + result.append(Path(os.path.join(pod_root_dir, "stack"))) return result -- 2.45.2 From ec3d6d1bc8b925f6da2d7ae8e71bee6641bf64ed Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 27 Oct 2023 13:48:46 -0500 Subject: [PATCH 2/3] Fix indent --- app/util.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/util.py b/app/util.py index 8aa751c5..d3b733a2 100644 --- a/app/util.py +++ b/app/util.py @@ -84,11 +84,11 @@ def get_plugin_code_paths(stack): pods = parsed_stack["pods"] result = [] for pod in pods: - if type(pod) is str: - result.append(get_stack_file_path(stack).parent) - else: - pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"]) - result.append(Path(os.path.join(pod_root_dir, "stack"))) + if type(pod) is str: + result.append(get_stack_file_path(stack).parent) + else: + pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"]) + result.append(Path(os.path.join(pod_root_dir, "stack"))) return result -- 2.45.2 From d33bc8ab0247196fae0732868637abc2e5d0f02f Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 27 Oct 2023 13:54:28 -0500 Subject: [PATCH 3/3] flake8 --- app/deploy/deployment_create.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/deploy/deployment_create.py b/app/deploy/deployment_create.py index b32d95f4..a7cbe57e 100644 --- a/app/deploy/deployment_create.py +++ b/app/deploy/deployment_create.py @@ -121,7 +121,7 @@ def call_stack_deploy_init(deploy_command_context): # Link with the python file in the stack # Call a function in it # If no function found, return None - python_file_paths = _commands_plugin_paths(deploy_command_context) + python_file_paths = _commands_plugin_paths(deploy_command_context) ret = None init_done = False @@ -135,7 +135,7 @@ def call_stack_deploy_init(deploy_command_context): ret = imported_stack.init(deploy_command_context) init_done = True else: - #TODO: remove this restriction + # TODO: remove this restriction print(f"Skipping init() from plugin {python_file_path}. Only one init() is allowed.") return ret -- 2.45.2