snowball/stack-orchestrator
5
0
Fork 0
forked from cerc-io/stack-orchestrator
Code Pull Requests Activity

Split act-runner into its own pod and offer as a distinct stack. (#612)

Browse Source 
* Split act-runner into its own pod and offer as a distinct stack.
This commit is contained in:
Thomas E Lackey 2023-10-27 13:57:13 -05:00 committed by GitHub
parent 6130eab5cb
commit 8cac598679
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 85 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/data/container-build/cerc-act-runner-task-executor/build.sh
View File

@ -2,4 +2,4 @@
# Build a local version of the task executor for act-runner
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/gitea/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR}
docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/act-runner/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR}

15
app/data/stacks/act-runner/README.md Normal file
View File

@ -0,0 +1,15 @@
# act-runner stack
## Example
```
$ laconic-so --stack act-runner deploy init --output act-runner.yml
$ laconic-so --stack act-runner deploy create --spec-file act-runner.yml --deployment-dir ~/opt/deployments/act-runner-1
$ echo "CERC_GITEA_RUNNER_REGISTRATION_TOKEN=FOO" >> ~/opt/deployments/act-runner-1/config.env
$ laconic-so deployment --dir ~/opt/deployments/act-runner-1 up
$ laconic-so --stack act-runner deploy create --spec-file act-runner.yml --deployment-dir ~/opt/deployments/act-runner-2
$ echo "CERC_GITEA_RUNNER_REGISTRATION_TOKEN=BAR" >> ~/opt/deployments/act-runner-2/config.env
$ laconic-so deployment --dir ~/opt/deployments/act-runner-2 up
```

15
app/data/stacks/act-runner/stack.yml Normal file
View File

@ -0,0 +1,15 @@
version: "1.1"
name: act-runner
description: "Local act-runner"
repos:
- git.vdb.to/cerc-io/hosting
- gitea.com/gitea/act_runner
containers:
- cerc/act-runner
- cerc/act-runner-task-executor
pods:
- name: act-runner
repository: cerc-io/hosting
path: act-runner
pre_start_command: "pre_start.sh"
post_start_command: "post_start.sh"

5
app/data/stacks/package-registry/stack.yml
View File

@ -13,3 +13,8 @@ pods:
path: gitea
pre_start_command: "run-this-first.sh"
post_start_command: "initialize-gitea.sh"
- name: act-runner
repository: cerc-io/hosting
path: act-runner
pre_start_command: "pre_start.sh"
post_start_command: "post_start.sh"

4
app/deploy/deploy.py
View File

@ -307,7 +307,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml")
else:
if deployment:
compose_file_name = os.path.join(compose_dir, "docker-compose.yml")
compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml")
pod_pre_start_command = pod["pre_start_command"]
pod_post_start_command = pod["post_start_command"]
script_dir = compose_dir.parent.joinpath("pods", pod_name, "scripts")
@ -317,7 +317,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
post_start_commands.append(os.path.join(script_dir, pod_post_start_command))
else:
pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
compose_file_name = os.path.join(pod_root_dir, "docker-compose.yml")
compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml")
pod_pre_start_command = pod["pre_start_command"]
pod_post_start_command = pod["post_start_command"]
if pod_pre_start_command is not None:

72
app/deploy/deployment_create.py
View File

@ -22,7 +22,7 @@ import random
from shutil import copy, copyfile, copytree
import sys
from app.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml,
get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_path)
get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_paths)
from app.deploy.deploy_types import DeploymentContext, DeployCommandContext, LaconicStackSetupCommand
@ -106,9 +106,10 @@ def _fixup_pod_file(pod, spec, compose_dir):
pod["services"][container_name]["ports"] = container_ports
def _commands_plugin_path(ctx: DeployCommandContext):
plugin_path = get_plugin_code_path(ctx.stack)
return plugin_path.joinpath("deploy", "commands.py")
def _commands_plugin_paths(ctx: DeployCommandContext):
plugin_paths = get_plugin_code_paths(ctx.stack)
ret = [p.joinpath("deploy", "commands.py") for p in plugin_paths]
return ret
# See: https://stackoverflow.com/a/54625079/1701505
@ -120,15 +121,23 @@ def call_stack_deploy_init(deploy_command_context):
# Link with the python file in the stack
# Call a function in it
# If no function found, return None
python_file_path = _commands_plugin_path(deploy_command_context)
if python_file_path.exists():
spec = util.spec_from_file_location("commands", python_file_path)
imported_stack = util.module_from_spec(spec)
spec.loader.exec_module(imported_stack)
if _has_method(imported_stack, "init"):
return imported_stack.init(deploy_command_context)
else:
return None
python_file_paths = _commands_plugin_paths(deploy_command_context)
ret = None
init_done = False
for python_file_path in python_file_paths:
if python_file_path.exists():
spec = util.spec_from_file_location("commands", python_file_path)
imported_stack = util.module_from_spec(spec)
spec.loader.exec_module(imported_stack)
if _has_method(imported_stack, "init"):
if not init_done:
ret = imported_stack.init(deploy_command_context)
init_done = True
else:
# TODO: remove this restriction
print(f"Skipping init() from plugin {python_file_path}. Only one init() is allowed.")
return ret
# TODO: fold this with function above
@ -136,16 +145,14 @@ def call_stack_deploy_setup(deploy_command_context, parameters: LaconicStackSetu
# Link with the python file in the stack
# Call a function in it
# If no function found, return None
python_file_path = _commands_plugin_path(deploy_command_context)
print(f"Path: {python_file_path}")
if python_file_path.exists():
spec = util.spec_from_file_location("commands", python_file_path)
imported_stack = util.module_from_spec(spec)
spec.loader.exec_module(imported_stack)
if _has_method(imported_stack, "setup"):
return imported_stack.setup(deploy_command_context, parameters, extra_args)
else:
return None
python_file_paths = _commands_plugin_paths(deploy_command_context)
for python_file_path in python_file_paths:
if python_file_path.exists():
spec = util.spec_from_file_location("commands", python_file_path)
imported_stack = util.module_from_spec(spec)
spec.loader.exec_module(imported_stack)
if _has_method(imported_stack, "setup"):
imported_stack.setup(deploy_command_context, parameters, extra_args)
# TODO: fold this with function above
@ -153,15 +160,14 @@ def call_stack_deploy_create(deployment_context, extra_args):
# Link with the python file in the stack
# Call a function in it
# If no function found, return None
python_file_path = _commands_plugin_path(deployment_context.command_context)
if python_file_path.exists():
spec = util.spec_from_file_location("commands", python_file_path)
imported_stack = util.module_from_spec(spec)
spec.loader.exec_module(imported_stack)
if _has_method(imported_stack, "create"):
return imported_stack.create(deployment_context, extra_args)
else:
return None
python_file_paths = _commands_plugin_paths(deployment_context.command_context)
for python_file_path in python_file_paths:
if python_file_path.exists():
spec = util.spec_from_file_location("commands", python_file_path)
imported_stack = util.module_from_spec(spec)
spec.loader.exec_module(imported_stack)
if _has_method(imported_stack, "create"):
imported_stack.create(deployment_context, extra_args)
# Inspect the pod yaml to find config files referenced in subdirectories
@ -336,7 +342,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
if global_options(ctx).debug:
print(f"extra config dirs: {extra_config_dirs}")
_fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir)
with open(os.path.join(destination_compose_dir, os.path.basename(pod_file_path)), "w") as output_file:
with open(os.path.join(destination_compose_dir, "docker-compose-%s.yml" % pod), "w") as output_file:
yaml.dump(parsed_pod_file, output_file)
# Copy the config files for the pod, if any
config_dirs = {pod}

16
app/util.py
View File

@ -79,16 +79,16 @@ def get_pod_list(parsed_stack):
return result
def get_plugin_code_path(stack):
def get_plugin_code_paths(stack):
parsed_stack = get_parsed_stack_config(stack)
pods = parsed_stack["pods"]
# TODO: Hack
pod = pods[0]
if type(pod) is str:
result = get_stack_file_path(stack).parent
else:
pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"])
result = Path(os.path.join(pod_root_dir, "stack"))
result = []
for pod in pods:
if type(pod) is str:
result.append(get_stack_file_path(stack).parent)
else:
pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"])
result.append(Path(os.path.join(pod_root_dir, "stack")))
return result