Archived
Rename app -> stack_orchestrator (#625)
This commit was merged in pull request #625.
This commit is contained in:
@@ -0,0 +1,79 @@
|
||||
# Copyright © 2022, 2023 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
from pathlib import Path
|
||||
from python_on_whales import DockerClient, DockerException
|
||||
from stack_orchestrator.deploy.deployer import Deployer, DeployerException, DeployerConfigGenerator
|
||||
|
||||
|
||||
class DockerDeployer(Deployer):
|
||||
name: str = "compose"
|
||||
|
||||
def __init__(self, compose_files, compose_project_name, compose_env_file) -> None:
|
||||
self.docker = DockerClient(compose_files=compose_files, compose_project_name=compose_project_name,
|
||||
compose_env_file=compose_env_file)
|
||||
|
||||
def up(self, detach, services):
|
||||
try:
|
||||
return self.docker.compose.up(detach=detach, services=services)
|
||||
except DockerException as e:
|
||||
raise DeployerException(e)
|
||||
|
||||
def down(self, timeout, volumes):
|
||||
try:
|
||||
return self.docker.compose.down(timeout=timeout, volumes=volumes)
|
||||
except DockerException as e:
|
||||
raise DeployerException(e)
|
||||
|
||||
def ps(self):
|
||||
try:
|
||||
return self.docker.compose.ps()
|
||||
except DockerException as e:
|
||||
raise DeployerException(e)
|
||||
|
||||
def port(self, service, private_port):
|
||||
try:
|
||||
return self.docker.compose.port(service=service, private_port=private_port)
|
||||
except DockerException as e:
|
||||
raise DeployerException(e)
|
||||
|
||||
def execute(self, service, command, tty, envs):
|
||||
try:
|
||||
return self.docker.compose.execute(service=service, command=command, tty=tty, envs=envs)
|
||||
except DockerException as e:
|
||||
raise DeployerException(e)
|
||||
|
||||
def logs(self, services, tail, follow, stream):
|
||||
try:
|
||||
return self.docker.compose.logs(services=services, tail=tail, follow=follow, stream=stream)
|
||||
except DockerException as e:
|
||||
raise DeployerException(e)
|
||||
|
||||
def run(self, image, command, user, volumes, entrypoint=None):
|
||||
try:
|
||||
return self.docker.run(image=image, command=command, user=user, volumes=volumes, entrypoint=entrypoint)
|
||||
except DockerException as e:
|
||||
raise DeployerException(e)
|
||||
|
||||
|
||||
class DockerDeployerConfigGenerator(DeployerConfigGenerator):
|
||||
config_file_name: str = "kind-config.yml"
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
# Nothing needed at present for the docker deployer
|
||||
def generate(self, deployment_dir: Path):
|
||||
pass
|
||||
@@ -0,0 +1,433 @@
|
||||
# Copyright © 2022, 2023 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
# Deploys the system components using a deployer (either docker-compose or k8s)
|
||||
|
||||
import hashlib
|
||||
import copy
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from importlib import resources
|
||||
import subprocess
|
||||
import click
|
||||
from pathlib import Path
|
||||
from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path
|
||||
from stack_orchestrator.deploy.deployer import Deployer, DeployerException
|
||||
from stack_orchestrator.deploy.deployer_factory import getDeployer
|
||||
from stack_orchestrator.deploy.deploy_types import ClusterContext, DeployCommandContext
|
||||
from stack_orchestrator.deploy.deployment_create import create as deployment_create
|
||||
from stack_orchestrator.deploy.deployment_create import init as deployment_init
|
||||
from stack_orchestrator.deploy.deployment_create import setup as deployment_setup
|
||||
|
||||
|
||||
@click.group()
|
||||
@click.option("--include", help="only start these components")
|
||||
@click.option("--exclude", help="don\'t start these components")
|
||||
@click.option("--env-file", help="env file to be used")
|
||||
@click.option("--cluster", help="specify a non-default cluster name")
|
||||
@click.option("--deploy-to", help="cluster system to deploy to (compose or k8s)")
|
||||
@click.pass_context
|
||||
def command(ctx, include, exclude, env_file, cluster, deploy_to):
|
||||
'''deploy a stack'''
|
||||
|
||||
# Although in theory for some subcommands (e.g. deploy create) the stack can be inferred,
|
||||
# Click doesn't allow us to know that here, so we make providing the stack mandatory
|
||||
stack = global_options2(ctx).stack
|
||||
if not stack:
|
||||
print("Error: --stack option is required")
|
||||
sys.exit(1)
|
||||
|
||||
if ctx.parent.obj.debug:
|
||||
print(f"ctx.parent.obj: {ctx.parent.obj}")
|
||||
|
||||
if deploy_to is None:
|
||||
deploy_to = "compose"
|
||||
|
||||
ctx.obj = create_deploy_context(global_options2(ctx), stack, include, exclude, cluster, env_file, deploy_to)
|
||||
# Subcommand is executed now, by the magic of click
|
||||
|
||||
|
||||
def create_deploy_context(global_context, stack, include, exclude, cluster, env_file, deployer):
|
||||
cluster_context = _make_cluster_context(global_context, stack, include, exclude, cluster, env_file)
|
||||
# See: https://gabrieldemarmiesse.github.io/python-on-whales/sub-commands/compose/
|
||||
deployer = getDeployer(deployer, compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster,
|
||||
compose_env_file=cluster_context.env_file)
|
||||
return DeployCommandContext(stack, cluster_context, deployer)
|
||||
|
||||
|
||||
def up_operation(ctx, services_list, stay_attached=False):
|
||||
global_context = ctx.parent.parent.obj
|
||||
deploy_context = ctx.obj
|
||||
if not global_context.dry_run:
|
||||
cluster_context = deploy_context.cluster_context
|
||||
container_exec_env = _make_runtime_env(global_context)
|
||||
for attr, value in container_exec_env.items():
|
||||
os.environ[attr] = value
|
||||
if global_context.verbose:
|
||||
print(f"Running compose up with container_exec_env: {container_exec_env}, extra_args: {services_list}")
|
||||
for pre_start_command in cluster_context.pre_start_commands:
|
||||
_run_command(global_context, cluster_context.cluster, pre_start_command)
|
||||
deploy_context.deployer.up(detach=not stay_attached, services=services_list)
|
||||
for post_start_command in cluster_context.post_start_commands:
|
||||
_run_command(global_context, cluster_context.cluster, post_start_command)
|
||||
_orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.deployer, container_exec_env)
|
||||
|
||||
|
||||
def down_operation(ctx, delete_volumes, extra_args_list):
|
||||
global_context = ctx.parent.parent.obj
|
||||
if not global_context.dry_run:
|
||||
if global_context.verbose:
|
||||
print("Running compose down")
|
||||
timeout_arg = None
|
||||
if extra_args_list:
|
||||
timeout_arg = extra_args_list[0]
|
||||
# Specify shutdown timeout (default 10s) to give services enough time to shutdown gracefully
|
||||
ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes)
|
||||
|
||||
|
||||
def ps_operation(ctx):
|
||||
global_context = ctx.parent.parent.obj
|
||||
if not global_context.dry_run:
|
||||
if global_context.verbose:
|
||||
print("Running compose ps")
|
||||
container_list = ctx.obj.deployer.ps()
|
||||
if len(container_list) > 0:
|
||||
print("Running containers:")
|
||||
for container in container_list:
|
||||
print(f"id: {container.id}, name: {container.name}, ports: ", end="")
|
||||
ports = container.network_settings.ports
|
||||
comma = ""
|
||||
for port_mapping in ports.keys():
|
||||
mapping = ports[port_mapping]
|
||||
print(comma, end="")
|
||||
if mapping is None:
|
||||
print(f"{port_mapping}", end="")
|
||||
else:
|
||||
print(f"{mapping[0]['HostIp']}:{mapping[0]['HostPort']}->{port_mapping}", end="")
|
||||
comma = ", "
|
||||
print()
|
||||
else:
|
||||
print("No containers running")
|
||||
|
||||
|
||||
def port_operation(ctx, extra_args):
|
||||
global_context = ctx.parent.parent.obj
|
||||
extra_args_list = list(extra_args) or None
|
||||
if not global_context.dry_run:
|
||||
if extra_args_list is None or len(extra_args_list) < 2:
|
||||
print("Usage: port <service> <exposed-port>")
|
||||
sys.exit(1)
|
||||
service_name = extra_args_list[0]
|
||||
exposed_port = extra_args_list[1]
|
||||
if global_context.verbose:
|
||||
print(f"Running compose port {service_name} {exposed_port}")
|
||||
mapped_port_data = ctx.obj.deployer.port(service_name, exposed_port)
|
||||
print(f"{mapped_port_data[0]}:{mapped_port_data[1]}")
|
||||
|
||||
|
||||
def exec_operation(ctx, extra_args):
|
||||
global_context = ctx.parent.parent.obj
|
||||
extra_args_list = list(extra_args) or None
|
||||
if not global_context.dry_run:
|
||||
if extra_args_list is None or len(extra_args_list) < 2:
|
||||
print("Usage: exec <service> <cmd>")
|
||||
sys.exit(1)
|
||||
service_name = extra_args_list[0]
|
||||
command_to_exec = ["sh", "-c"] + extra_args_list[1:]
|
||||
container_exec_env = _make_runtime_env(global_context)
|
||||
if global_context.verbose:
|
||||
print(f"Running compose exec {service_name} {command_to_exec}")
|
||||
try:
|
||||
ctx.obj.deployer.execute(service_name, command_to_exec, envs=container_exec_env)
|
||||
except DeployerException:
|
||||
print("container command returned error exit status")
|
||||
|
||||
|
||||
def logs_operation(ctx, tail: int, follow: bool, extra_args: str):
|
||||
global_context = ctx.parent.parent.obj
|
||||
extra_args_list = list(extra_args) or None
|
||||
if not global_context.dry_run:
|
||||
if global_context.verbose:
|
||||
print("Running compose logs")
|
||||
services_list = extra_args_list if extra_args_list is not None else []
|
||||
logs_stream = ctx.obj.deployer.logs(services=services_list, tail=tail, follow=follow, stream=True)
|
||||
for stream_type, stream_content in logs_stream:
|
||||
print(stream_content.decode("utf-8"), end="")
|
||||
|
||||
|
||||
@command.command()
|
||||
@click.argument('extra_args', nargs=-1) # help: command: up <service1> <service2>
|
||||
@click.pass_context
|
||||
def up(ctx, extra_args):
|
||||
extra_args_list = list(extra_args) or None
|
||||
up_operation(ctx, extra_args_list)
|
||||
|
||||
|
||||
@command.command()
|
||||
@click.option("--delete-volumes/--preserve-volumes", default=False, help="delete data volumes")
|
||||
@click.argument('extra_args', nargs=-1) # help: command: down<service1> <service2>
|
||||
@click.pass_context
|
||||
def down(ctx, delete_volumes, extra_args):
|
||||
extra_args_list = list(extra_args) or None
|
||||
down_operation(ctx, delete_volumes, extra_args_list)
|
||||
|
||||
|
||||
@command.command()
|
||||
@click.pass_context
|
||||
def ps(ctx):
|
||||
ps_operation(ctx)
|
||||
|
||||
|
||||
@command.command()
|
||||
@click.argument('extra_args', nargs=-1) # help: command: port <service1> <service2>
|
||||
@click.pass_context
|
||||
def port(ctx, extra_args):
|
||||
port_operation(ctx, extra_args)
|
||||
|
||||
|
||||
@command.command()
|
||||
@click.argument('extra_args', nargs=-1) # help: command: exec <service> <command>
|
||||
@click.pass_context
|
||||
def exec(ctx, extra_args):
|
||||
exec_operation(ctx, extra_args)
|
||||
|
||||
|
||||
@command.command()
|
||||
@click.option("--tail", "-n", default=None, help="number of lines to display")
|
||||
@click.option("--follow", "-f", is_flag=True, default=False, help="follow log output")
|
||||
@click.argument('extra_args', nargs=-1) # help: command: logs <service1> <service2>
|
||||
@click.pass_context
|
||||
def logs(ctx, tail, follow, extra_args):
|
||||
logs_operation(ctx, tail, follow, extra_args)
|
||||
|
||||
|
||||
def get_stack_status(ctx, stack):
|
||||
|
||||
ctx_copy = copy.copy(ctx)
|
||||
ctx_copy.stack = stack
|
||||
|
||||
cluster_context = _make_cluster_context(ctx_copy, stack, None, None, None, None)
|
||||
deployer = Deployer(compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster)
|
||||
# TODO: refactor to avoid duplicating this code above
|
||||
if ctx.verbose:
|
||||
print("Running compose ps")
|
||||
container_list = deployer.ps()
|
||||
if len(container_list) > 0:
|
||||
if ctx.debug:
|
||||
print(f"Container list from compose ps: {container_list}")
|
||||
return True
|
||||
else:
|
||||
if ctx.debug:
|
||||
print("No containers found from compose ps")
|
||||
False
|
||||
|
||||
|
||||
def _make_runtime_env(ctx):
|
||||
container_exec_env = {
|
||||
"CERC_HOST_UID": f"{os.getuid()}",
|
||||
"CERC_HOST_GID": f"{os.getgid()}"
|
||||
}
|
||||
container_exec_env.update({"CERC_SCRIPT_DEBUG": "true"} if ctx.debug else {})
|
||||
return container_exec_env
|
||||
|
||||
|
||||
# stack has to be either PathLike pointing to a stack yml file, or a string with the name of a known stack
|
||||
def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
|
||||
|
||||
dev_root_path = get_dev_root_path(ctx)
|
||||
|
||||
# TODO: huge hack, fix this
|
||||
# If the caller passed a path for the stack file, then we know that we can get the compose files
|
||||
# from the same directory
|
||||
deployment = False
|
||||
if isinstance(stack, os.PathLike):
|
||||
compose_dir = stack.parent.joinpath("compose")
|
||||
deployment = True
|
||||
else:
|
||||
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
|
||||
compose_dir = Path(__file__).absolute().parent.parent.joinpath("data", "compose")
|
||||
|
||||
if cluster is None:
|
||||
# Create default unique, stable cluster name from confile file path and stack name if provided
|
||||
# TODO: change this to the config file path
|
||||
path = os.path.realpath(sys.argv[0])
|
||||
unique_cluster_descriptor = f"{path},{stack},{include},{exclude}"
|
||||
if ctx.debug:
|
||||
print(f"pre-hash descriptor: {unique_cluster_descriptor}")
|
||||
hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()
|
||||
cluster = f"laconic-{hash}"
|
||||
if ctx.verbose:
|
||||
print(f"Using cluster name: {cluster}")
|
||||
|
||||
# See: https://stackoverflow.com/a/20885799/1701505
|
||||
from stack_orchestrator import data
|
||||
with resources.open_text(data, "pod-list.txt") as pod_list_file:
|
||||
all_pods = pod_list_file.read().splitlines()
|
||||
|
||||
pods_in_scope = []
|
||||
if stack:
|
||||
stack_config = get_parsed_stack_config(stack)
|
||||
# TODO: syntax check the input here
|
||||
pods_in_scope = stack_config['pods']
|
||||
cluster_config = stack_config['config'] if 'config' in stack_config else None
|
||||
else:
|
||||
pods_in_scope = all_pods
|
||||
cluster_config = None
|
||||
|
||||
# Convert all pod definitions to v1.1 format
|
||||
pods_in_scope = _convert_to_new_format(pods_in_scope)
|
||||
|
||||
if ctx.verbose:
|
||||
print(f"Pods: {pods_in_scope}")
|
||||
|
||||
# Construct a docker compose command suitable for our purpose
|
||||
|
||||
compose_files = []
|
||||
pre_start_commands = []
|
||||
post_start_commands = []
|
||||
for pod in pods_in_scope:
|
||||
pod_name = pod["name"]
|
||||
pod_repository = pod["repository"]
|
||||
pod_path = pod["path"]
|
||||
if include_exclude_check(pod_name, include, exclude):
|
||||
if pod_repository is None or pod_repository == "internal":
|
||||
compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml")
|
||||
else:
|
||||
if deployment:
|
||||
compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml")
|
||||
pod_pre_start_command = pod["pre_start_command"]
|
||||
pod_post_start_command = pod["post_start_command"]
|
||||
script_dir = compose_dir.parent.joinpath("pods", pod_name, "scripts")
|
||||
if pod_pre_start_command is not None:
|
||||
pre_start_commands.append(os.path.join(script_dir, pod_pre_start_command))
|
||||
if pod_post_start_command is not None:
|
||||
post_start_commands.append(os.path.join(script_dir, pod_post_start_command))
|
||||
else:
|
||||
pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
|
||||
compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml")
|
||||
pod_pre_start_command = pod["pre_start_command"]
|
||||
pod_post_start_command = pod["post_start_command"]
|
||||
if pod_pre_start_command is not None:
|
||||
pre_start_commands.append(os.path.join(pod_root_dir, pod_pre_start_command))
|
||||
if pod_post_start_command is not None:
|
||||
post_start_commands.append(os.path.join(pod_root_dir, pod_post_start_command))
|
||||
compose_files.append(compose_file_name)
|
||||
else:
|
||||
if ctx.verbose:
|
||||
print(f"Excluding: {pod_name}")
|
||||
|
||||
if ctx.verbose:
|
||||
print(f"files: {compose_files}")
|
||||
|
||||
return ClusterContext(ctx, cluster, compose_files, pre_start_commands, post_start_commands, cluster_config, env_file)
|
||||
|
||||
|
||||
def _convert_to_new_format(old_pod_array):
|
||||
new_pod_array = []
|
||||
for old_pod in old_pod_array:
|
||||
if isinstance(old_pod, dict):
|
||||
new_pod_array.append(old_pod)
|
||||
else:
|
||||
new_pod = {
|
||||
"name": old_pod,
|
||||
"repository": "internal",
|
||||
"path": old_pod
|
||||
}
|
||||
new_pod_array.append(new_pod)
|
||||
return new_pod_array
|
||||
|
||||
|
||||
def _run_command(ctx, cluster_name, command):
|
||||
if ctx.verbose:
|
||||
print(f"Running command: {command}")
|
||||
command_dir = os.path.dirname(command)
|
||||
command_file = os.path.join(".", os.path.basename(command))
|
||||
command_env = os.environ.copy()
|
||||
command_env["CERC_SO_COMPOSE_PROJECT"] = cluster_name
|
||||
if ctx.debug:
|
||||
command_env["CERC_SCRIPT_DEBUG"] = "true"
|
||||
command_result = subprocess.run(command_file, shell=True, env=command_env, cwd=command_dir)
|
||||
if command_result.returncode != 0:
|
||||
print(f"FATAL Error running command: {command}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _orchestrate_cluster_config(ctx, cluster_config, deployer, container_exec_env):
|
||||
|
||||
@dataclass
|
||||
class ConfigDirective:
|
||||
source_container: str
|
||||
source_variable: str
|
||||
destination_container: str
|
||||
destination_variable: str
|
||||
|
||||
if cluster_config is not None:
|
||||
for container in cluster_config:
|
||||
container_config = cluster_config[container]
|
||||
if ctx.verbose:
|
||||
print(f"{container} config: {container_config}")
|
||||
for directive in container_config:
|
||||
pd = ConfigDirective(
|
||||
container_config[directive].split(".")[0],
|
||||
container_config[directive].split(".")[1],
|
||||
container,
|
||||
directive
|
||||
)
|
||||
if ctx.verbose:
|
||||
print(f"Setting {pd.destination_container}.{pd.destination_variable}"
|
||||
f" = {pd.source_container}.{pd.source_variable}")
|
||||
# TODO: add a timeout
|
||||
waiting_for_data = True
|
||||
destination_output = "*** no output received yet ***"
|
||||
while waiting_for_data:
|
||||
# TODO: fix the script paths so they're consistent between containers
|
||||
source_value = None
|
||||
try:
|
||||
source_value = deployer.execute(pd.source_container,
|
||||
["sh", "-c",
|
||||
"sh /docker-entrypoint-scripts.d/export-"
|
||||
f"{pd.source_variable}.sh"],
|
||||
tty=False,
|
||||
envs=container_exec_env)
|
||||
except DeployerException as error:
|
||||
if ctx.debug:
|
||||
print(f"Docker exception reading config source: {error}")
|
||||
# If the script executed failed for some reason, we get:
|
||||
# "It returned with code 1"
|
||||
if "It returned with code 1" in str(error):
|
||||
if ctx.verbose:
|
||||
print("Config export script returned an error, re-trying")
|
||||
# If the script failed to execute (e.g. the file is not there) then we get:
|
||||
# "It returned with code 2"
|
||||
if "It returned with code 2" in str(error):
|
||||
print(f"Fatal error reading config source: {error}")
|
||||
if source_value:
|
||||
if ctx.debug:
|
||||
print(f"fetched source value: {source_value}")
|
||||
destination_output = deployer.execute(pd.destination_container,
|
||||
["sh", "-c",
|
||||
f"sh /scripts/import-{pd.destination_variable}.sh"
|
||||
f" {source_value}"],
|
||||
tty=False,
|
||||
envs=container_exec_env)
|
||||
waiting_for_data = False
|
||||
if ctx.debug:
|
||||
print(f"destination output: {destination_output}")
|
||||
|
||||
|
||||
command.add_command(deployment_init)
|
||||
command.add_command(deployment_create)
|
||||
command.add_command(deployment_setup)
|
||||
@@ -0,0 +1,68 @@
|
||||
# Copyright © 2023 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
from typing import List
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from stack_orchestrator.command_types import CommandOptions
|
||||
from stack_orchestrator.deploy.deployer import Deployer
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClusterContext:
|
||||
options: CommandOptions # TODO: this should be in its own object not stuffed in here
|
||||
cluster: str
|
||||
compose_files: List[str]
|
||||
pre_start_commands: List[str]
|
||||
post_start_commands: List[str]
|
||||
config: str
|
||||
env_file: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeployCommandContext:
|
||||
stack: str
|
||||
cluster_context: ClusterContext
|
||||
deployer: Deployer
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeploymentContext:
|
||||
deployment_dir: Path
|
||||
command_context: DeployCommandContext
|
||||
|
||||
|
||||
@dataclass
|
||||
class VolumeMapping:
|
||||
host_path: str
|
||||
container_path: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class LaconicStackSetupCommand:
|
||||
chain_id: str
|
||||
node_moniker: str
|
||||
key_name: str
|
||||
initialize_network: bool
|
||||
join_network: bool
|
||||
create_network: bool
|
||||
gentx_file_list: str
|
||||
genesis_file: str
|
||||
network_dir: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class LaconicStackCreateCommand:
|
||||
network_dir: str
|
||||
@@ -0,0 +1,62 @@
|
||||
# Copyright © 2022, 2023 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
from typing import List
|
||||
from stack_orchestrator.deploy.deploy_types import DeployCommandContext, VolumeMapping
|
||||
from stack_orchestrator.util import get_parsed_stack_config, get_yaml, get_compose_file_dir, get_pod_list
|
||||
|
||||
|
||||
def _container_image_from_service(stack: str, service: str):
|
||||
# Parse the compose files looking for the image name of the specified service
|
||||
image_name = None
|
||||
parsed_stack = get_parsed_stack_config(stack)
|
||||
pods = get_pod_list(parsed_stack)
|
||||
yaml = get_yaml()
|
||||
for pod in pods:
|
||||
pod_file_path = os.path.join(get_compose_file_dir(), f"docker-compose-{pod}.yml")
|
||||
parsed_pod_file = yaml.load(open(pod_file_path, "r"))
|
||||
if "services" in parsed_pod_file:
|
||||
services = parsed_pod_file["services"]
|
||||
if service in services:
|
||||
service_definition = services[service]
|
||||
if "image" in service_definition:
|
||||
image_name = service_definition["image"]
|
||||
return image_name
|
||||
|
||||
|
||||
def _volumes_to_docker(mounts: List[VolumeMapping]):
|
||||
# Example from doc: [("/", "/host"), ("/etc/hosts", "/etc/hosts", "rw")]
|
||||
result = []
|
||||
for mount in mounts:
|
||||
docker_volume = (mount.host_path, mount.container_path)
|
||||
result.append(docker_volume)
|
||||
return result
|
||||
|
||||
|
||||
def run_container_command(ctx: DeployCommandContext, service: str, command: str, mounts: List[VolumeMapping]):
|
||||
deployer = ctx.deployer
|
||||
container_image = _container_image_from_service(ctx.stack, service)
|
||||
docker_volumes = _volumes_to_docker(mounts)
|
||||
if ctx.cluster_context.options.debug:
|
||||
print(f"Running this command in {service} container: {command}")
|
||||
docker_output = deployer.run(
|
||||
container_image,
|
||||
["-c", command], entrypoint="sh",
|
||||
user=f"{os.getuid()}:{os.getgid()}",
|
||||
volumes=docker_volumes
|
||||
)
|
||||
# There doesn't seem to be a way to get an exit code from docker.run()
|
||||
return (docker_output, 0)
|
||||
@@ -0,0 +1,60 @@
|
||||
# Copyright © 2023 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class Deployer(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def up(self, detach, services):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def down(self, timeout, volumes):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def ps(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def port(self, service, private_port):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def execute(self, service_name, command, tty, envs):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def logs(self, services, tail, follow, stream):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def run(self, image, command, user, volumes, entrypoint):
|
||||
pass
|
||||
|
||||
|
||||
class DeployerException(Exception):
|
||||
def __init__(self, *args: object) -> None:
|
||||
super().__init__(*args)
|
||||
|
||||
|
||||
class DeployerConfigGenerator(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def generate(self, deployment_dir: Path):
|
||||
pass
|
||||
@@ -0,0 +1,35 @@
|
||||
# Copyright © 2023 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
from stack_orchestrator.deploy.k8s.deploy_k8s import K8sDeployer, K8sDeployerConfigGenerator
|
||||
from stack_orchestrator.deploy.compose.deploy_docker import DockerDeployer, DockerDeployerConfigGenerator
|
||||
|
||||
|
||||
def getDeployerConfigGenerator(type: str):
|
||||
if type == "compose" or type is None:
|
||||
return DockerDeployerConfigGenerator()
|
||||
elif type == "k8s":
|
||||
return K8sDeployerConfigGenerator()
|
||||
else:
|
||||
print(f"ERROR: deploy-to {type} is not valid")
|
||||
|
||||
|
||||
def getDeployer(type: str, compose_files, compose_project_name, compose_env_file):
|
||||
if type == "compose" or type is None:
|
||||
return DockerDeployer(compose_files, compose_project_name, compose_env_file)
|
||||
elif type == "k8s":
|
||||
return K8sDeployer(compose_files, compose_project_name, compose_env_file)
|
||||
else:
|
||||
print(f"ERROR: deploy-to {type} is not valid")
|
||||
@@ -0,0 +1,162 @@
|
||||
# Copyright © 2022, 2023 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
import click
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation
|
||||
from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context
|
||||
from stack_orchestrator.deploy.stack import Stack
|
||||
from stack_orchestrator.deploy.spec import Spec
|
||||
|
||||
|
||||
class DeploymentContext:
|
||||
dir: Path
|
||||
spec: Spec
|
||||
stack: Stack
|
||||
|
||||
def get_stack_file(self):
|
||||
return self.dir.joinpath("stack.yml")
|
||||
|
||||
def get_spec_file(self):
|
||||
return self.dir.joinpath("spec.yml")
|
||||
|
||||
def get_env_file(self):
|
||||
return self.dir.joinpath("config.env")
|
||||
|
||||
# TODO: implement me
|
||||
def get_cluster_name(self):
|
||||
return None
|
||||
|
||||
def init(self, dir):
|
||||
self.dir = dir
|
||||
self.stack = Stack()
|
||||
self.stack.init_from_file(self.get_stack_file())
|
||||
self.spec = Spec()
|
||||
self.spec.init_from_file(self.get_spec_file())
|
||||
|
||||
|
||||
@click.group()
|
||||
@click.option("--dir", required=True, help="path to deployment directory")
|
||||
@click.pass_context
|
||||
def command(ctx, dir):
|
||||
'''manage a deployment'''
|
||||
|
||||
# Check that --stack wasn't supplied
|
||||
if ctx.parent.obj.stack:
|
||||
print("Error: --stack can't be supplied with the deployment command")
|
||||
sys.exit(1)
|
||||
# Check dir is valid
|
||||
dir_path = Path(dir)
|
||||
if not dir_path.exists():
|
||||
print(f"Error: deployment directory {dir} does not exist")
|
||||
sys.exit(1)
|
||||
if not dir_path.is_dir():
|
||||
print(f"Error: supplied deployment directory path {dir} exists but is a file not a directory")
|
||||
sys.exit(1)
|
||||
# Store the deployment context for subcommands
|
||||
deployment_context = DeploymentContext()
|
||||
deployment_context.init(dir_path)
|
||||
ctx.obj = deployment_context
|
||||
|
||||
|
||||
def make_deploy_context(ctx):
|
||||
context: DeploymentContext = ctx.obj
|
||||
stack_file_path = context.get_stack_file()
|
||||
env_file = context.get_env_file()
|
||||
cluster_name = context.get_cluster_name()
|
||||
return create_deploy_context(ctx.parent.parent.obj, stack_file_path, None, None, cluster_name, env_file,
|
||||
context.spec.obj["deploy-to"])
|
||||
|
||||
|
||||
@command.command()
|
||||
@click.option("--stay-attached/--detatch-terminal", default=False, help="detatch or not to see container stdout")
|
||||
@click.argument('extra_args', nargs=-1) # help: command: up <service1> <service2>
|
||||
@click.pass_context
|
||||
def up(ctx, stay_attached, extra_args):
|
||||
ctx.obj = make_deploy_context(ctx)
|
||||
services_list = list(extra_args) or None
|
||||
up_operation(ctx, services_list, stay_attached)
|
||||
|
||||
|
||||
# start is the preferred alias for up
|
||||
@command.command()
|
||||
@click.option("--stay-attached/--detatch-terminal", default=False, help="detatch or not to see container stdout")
|
||||
@click.argument('extra_args', nargs=-1) # help: command: up <service1> <service2>
|
||||
@click.pass_context
|
||||
def start(ctx, stay_attached, extra_args):
|
||||
ctx.obj = make_deploy_context(ctx)
|
||||
services_list = list(extra_args) or None
|
||||
up_operation(ctx, services_list, stay_attached)
|
||||
|
||||
|
||||
@command.command()
|
||||
@click.option("--delete-volumes/--preserve-volumes", default=False, help="delete data volumes")
|
||||
@click.argument('extra_args', nargs=-1) # help: command: down <service1> <service2>
|
||||
@click.pass_context
|
||||
def down(ctx, delete_volumes, extra_args):
|
||||
# Get the stack config file name
|
||||
# TODO: add cluster name and env file here
|
||||
ctx.obj = make_deploy_context(ctx)
|
||||
down_operation(ctx, delete_volumes, extra_args)
|
||||
|
||||
|
||||
# stop is the preferred alias for down
|
||||
@command.command()
|
||||
@click.option("--delete-volumes/--preserve-volumes", default=False, help="delete data volumes")
|
||||
@click.argument('extra_args', nargs=-1) # help: command: down <service1> <service2>
|
||||
@click.pass_context
|
||||
def stop(ctx, delete_volumes, extra_args):
|
||||
# TODO: add cluster name and env file here
|
||||
ctx.obj = make_deploy_context(ctx)
|
||||
down_operation(ctx, delete_volumes, extra_args)
|
||||
|
||||
|
||||
@command.command()
|
||||
@click.pass_context
|
||||
def ps(ctx):
|
||||
ctx.obj = make_deploy_context(ctx)
|
||||
ps_operation(ctx)
|
||||
|
||||
|
||||
@command.command()
|
||||
@click.argument('extra_args', nargs=-1) # help: command: port <service1> <service2>
|
||||
@click.pass_context
|
||||
def port(ctx, extra_args):
|
||||
port_operation(ctx, extra_args)
|
||||
|
||||
|
||||
@command.command()
|
||||
@click.argument('extra_args', nargs=-1) # help: command: exec <service> <command>
|
||||
@click.pass_context
|
||||
def exec(ctx, extra_args):
|
||||
ctx.obj = make_deploy_context(ctx)
|
||||
exec_operation(ctx, extra_args)
|
||||
|
||||
|
||||
@command.command()
|
||||
@click.option("--tail", "-n", default=None, help="number of lines to display")
|
||||
@click.option("--follow", "-f", is_flag=True, default=False, help="follow log output")
|
||||
@click.argument('extra_args', nargs=-1) # help: command: logs <service1> <service2>
|
||||
@click.pass_context
|
||||
def logs(ctx, tail, follow, extra_args):
|
||||
ctx.obj = make_deploy_context(ctx)
|
||||
logs_operation(ctx, tail, follow, extra_args)
|
||||
|
||||
|
||||
@command.command()
|
||||
@click.pass_context
|
||||
def status(ctx):
|
||||
print(f"Context: {ctx.parent.obj}")
|
||||
@@ -0,0 +1,397 @@
|
||||
# Copyright © 2022, 2023 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
import click
|
||||
from importlib import util
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
import random
|
||||
from shutil import copy, copyfile, copytree
|
||||
import sys
|
||||
from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
|
||||
global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
|
||||
get_pod_script_paths, get_plugin_code_paths)
|
||||
from stack_orchestrator.deploy.deploy_types import DeploymentContext, DeployCommandContext, LaconicStackSetupCommand
|
||||
from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator
|
||||
|
||||
|
||||
def _make_default_deployment_dir():
|
||||
return "deployment-001"
|
||||
|
||||
|
||||
def _get_ports(stack):
|
||||
ports = {}
|
||||
parsed_stack = get_parsed_stack_config(stack)
|
||||
pods = get_pod_list(parsed_stack)
|
||||
yaml = get_yaml()
|
||||
for pod in pods:
|
||||
pod_file_path = get_pod_file_path(parsed_stack, pod)
|
||||
parsed_pod_file = yaml.load(open(pod_file_path, "r"))
|
||||
if "services" in parsed_pod_file:
|
||||
for svc_name, svc in parsed_pod_file["services"].items():
|
||||
if "ports" in svc:
|
||||
# Ports can appear as strings or numbers. We normalize them as strings.
|
||||
ports[svc_name] = [str(x) for x in svc["ports"]]
|
||||
return ports
|
||||
|
||||
|
||||
def _get_named_volumes(stack):
|
||||
# Parse the compose files looking for named volumes
|
||||
named_volumes = []
|
||||
parsed_stack = get_parsed_stack_config(stack)
|
||||
pods = get_pod_list(parsed_stack)
|
||||
yaml = get_yaml()
|
||||
for pod in pods:
|
||||
pod_file_path = get_pod_file_path(parsed_stack, pod)
|
||||
parsed_pod_file = yaml.load(open(pod_file_path, "r"))
|
||||
if "volumes" in parsed_pod_file:
|
||||
volumes = parsed_pod_file["volumes"]
|
||||
for volume in volumes.keys():
|
||||
# Volume definition looks like:
|
||||
# 'laconicd-data': None
|
||||
named_volumes.append(volume)
|
||||
return named_volumes
|
||||
|
||||
|
||||
# If we're mounting a volume from a relatie path, then we
|
||||
# assume the directory doesn't exist yet and create it
|
||||
# so the deployment will start
|
||||
# Also warn if the path is absolute and doesn't exist
|
||||
def _create_bind_dir_if_relative(volume, path_string, compose_dir):
|
||||
path = Path(path_string)
|
||||
if not path.is_absolute():
|
||||
absolute_path = Path(compose_dir).parent.joinpath(path)
|
||||
absolute_path.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
if not path.exists():
|
||||
print(f"WARNING: mount path for volume {volume} does not exist: {path_string}")
|
||||
|
||||
|
||||
# See: https://stackoverflow.com/questions/45699189/editing-docker-compose-yml-with-pyyaml
|
||||
def _fixup_pod_file(pod, spec, compose_dir):
|
||||
# Fix up volumes
|
||||
if "volumes" in spec:
|
||||
spec_volumes = spec["volumes"]
|
||||
if "volumes" in pod:
|
||||
pod_volumes = pod["volumes"]
|
||||
for volume in pod_volumes.keys():
|
||||
if volume in spec_volumes:
|
||||
volume_spec = spec_volumes[volume]
|
||||
volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}"
|
||||
_create_bind_dir_if_relative(volume, volume_spec, compose_dir)
|
||||
new_volume_spec = {"driver": "local",
|
||||
"driver_opts": {
|
||||
"type": "none",
|
||||
"device": volume_spec_fixedup,
|
||||
"o": "bind"
|
||||
}
|
||||
}
|
||||
pod["volumes"][volume] = new_volume_spec
|
||||
# Fix up ports
|
||||
if "ports" in spec:
|
||||
spec_ports = spec["ports"]
|
||||
for container_name, container_ports in spec_ports.items():
|
||||
if container_name in pod["services"]:
|
||||
pod["services"][container_name]["ports"] = container_ports
|
||||
|
||||
|
||||
def _commands_plugin_paths(ctx: DeployCommandContext):
|
||||
plugin_paths = get_plugin_code_paths(ctx.stack)
|
||||
ret = [p.joinpath("deploy", "commands.py") for p in plugin_paths]
|
||||
return ret
|
||||
|
||||
|
||||
# See: https://stackoverflow.com/a/54625079/1701505
|
||||
def _has_method(o, name):
|
||||
return callable(getattr(o, name, None))
|
||||
|
||||
|
||||
def call_stack_deploy_init(deploy_command_context):
|
||||
# Link with the python file in the stack
|
||||
# Call a function in it
|
||||
# If no function found, return None
|
||||
python_file_paths = _commands_plugin_paths(deploy_command_context)
|
||||
|
||||
ret = None
|
||||
init_done = False
|
||||
for python_file_path in python_file_paths:
|
||||
if python_file_path.exists():
|
||||
spec = util.spec_from_file_location("commands", python_file_path)
|
||||
imported_stack = util.module_from_spec(spec)
|
||||
spec.loader.exec_module(imported_stack)
|
||||
if _has_method(imported_stack, "init"):
|
||||
if not init_done:
|
||||
ret = imported_stack.init(deploy_command_context)
|
||||
init_done = True
|
||||
else:
|
||||
# TODO: remove this restriction
|
||||
print(f"Skipping init() from plugin {python_file_path}. Only one init() is allowed.")
|
||||
return ret
|
||||
|
||||
|
||||
# TODO: fold this with function above
|
||||
def call_stack_deploy_setup(deploy_command_context, parameters: LaconicStackSetupCommand, extra_args):
|
||||
# Link with the python file in the stack
|
||||
# Call a function in it
|
||||
# If no function found, return None
|
||||
python_file_paths = _commands_plugin_paths(deploy_command_context)
|
||||
for python_file_path in python_file_paths:
|
||||
if python_file_path.exists():
|
||||
spec = util.spec_from_file_location("commands", python_file_path)
|
||||
imported_stack = util.module_from_spec(spec)
|
||||
spec.loader.exec_module(imported_stack)
|
||||
if _has_method(imported_stack, "setup"):
|
||||
imported_stack.setup(deploy_command_context, parameters, extra_args)
|
||||
|
||||
|
||||
# TODO: fold this with function above
|
||||
def call_stack_deploy_create(deployment_context, extra_args):
|
||||
# Link with the python file in the stack
|
||||
# Call a function in it
|
||||
# If no function found, return None
|
||||
python_file_paths = _commands_plugin_paths(deployment_context.command_context)
|
||||
for python_file_path in python_file_paths:
|
||||
if python_file_path.exists():
|
||||
spec = util.spec_from_file_location("commands", python_file_path)
|
||||
imported_stack = util.module_from_spec(spec)
|
||||
spec.loader.exec_module(imported_stack)
|
||||
if _has_method(imported_stack, "create"):
|
||||
imported_stack.create(deployment_context, extra_args)
|
||||
|
||||
|
||||
# Inspect the pod yaml to find config files referenced in subdirectories
|
||||
# other than the one associated with the pod
|
||||
def _find_extra_config_dirs(parsed_pod_file, pod):
|
||||
config_dirs = set()
|
||||
services = parsed_pod_file["services"]
|
||||
for service in services:
|
||||
service_info = services[service]
|
||||
if "volumes" in service_info:
|
||||
for volume in service_info["volumes"]:
|
||||
if ":" in volume:
|
||||
host_path = volume.split(":")[0]
|
||||
if host_path.startswith("../config"):
|
||||
config_dir = host_path.split("/")[2]
|
||||
if config_dir != pod:
|
||||
config_dirs.add(config_dir)
|
||||
return config_dirs
|
||||
|
||||
|
||||
def _get_mapped_ports(stack: str, map_recipe: str):
|
||||
port_map_recipes = ["any-variable-random", "localhost-same", "any-same", "localhost-fixed-random", "any-fixed-random"]
|
||||
ports = _get_ports(stack)
|
||||
if ports:
|
||||
# Implement any requested mapping recipe
|
||||
if map_recipe:
|
||||
if map_recipe in port_map_recipes:
|
||||
for service in ports.keys():
|
||||
ports_array = ports[service]
|
||||
for x in range(0, len(ports_array)):
|
||||
orig_port = ports_array[x]
|
||||
# Strip /udp suffix if present
|
||||
bare_orig_port = orig_port.replace("/udp", "")
|
||||
random_port = random.randint(20000, 50000) # Beware: we're relying on luck to not collide
|
||||
if map_recipe == "any-variable-random":
|
||||
# This is the default so take no action
|
||||
pass
|
||||
elif map_recipe == "localhost-same":
|
||||
# Replace instances of "- XX" with "- 127.0.0.1:XX"
|
||||
ports_array[x] = f"127.0.0.1:{bare_orig_port}:{orig_port}"
|
||||
elif map_recipe == "any-same":
|
||||
# Replace instances of "- XX" with "- 0.0.0.0:XX"
|
||||
ports_array[x] = f"0.0.0.0:{bare_orig_port}:{orig_port}"
|
||||
elif map_recipe == "localhost-fixed-random":
|
||||
# Replace instances of "- XX" with "- 127.0.0.1:<rnd>:XX"
|
||||
ports_array[x] = f"127.0.0.1:{random_port}:{orig_port}"
|
||||
elif map_recipe == "any-fixed-random":
|
||||
# Replace instances of "- XX" with "- 0.0.0.0:<rnd>:XX"
|
||||
ports_array[x] = f"0.0.0.0:{random_port}:{orig_port}"
|
||||
else:
|
||||
print("Error: bad map_recipe")
|
||||
else:
|
||||
print(f"Error: --map-ports-to-host must specify one of: {port_map_recipes}")
|
||||
sys.exit(1)
|
||||
return ports
|
||||
|
||||
|
||||
def _parse_config_variables(variable_values: str):
|
||||
result = None
|
||||
if variable_values:
|
||||
value_pairs = variable_values.split(",")
|
||||
if len(value_pairs):
|
||||
result_values = {}
|
||||
for value_pair in value_pairs:
|
||||
variable_value_pair = value_pair.split("=")
|
||||
if len(variable_value_pair) != 2:
|
||||
print(f"ERROR: config argument is not valid: {variable_values}")
|
||||
sys.exit(1)
|
||||
variable_name = variable_value_pair[0]
|
||||
variable_value = variable_value_pair[1]
|
||||
result_values[variable_name] = variable_value
|
||||
result = {"config": result_values}
|
||||
return result
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--config", help="Provide config variables for the deployment")
|
||||
@click.option("--output", required=True, help="Write yaml spec file here")
|
||||
@click.option("--map-ports-to-host", required=False,
|
||||
help="Map ports to the host as one of: any-variable-random (default), "
|
||||
"localhost-same, any-same, localhost-fixed-random, any-fixed-random")
|
||||
@click.pass_context
|
||||
def init(ctx, config, output, map_ports_to_host):
|
||||
yaml = get_yaml()
|
||||
stack = global_options(ctx).stack
|
||||
debug = global_options(ctx).debug
|
||||
default_spec_file_content = call_stack_deploy_init(ctx.obj)
|
||||
spec_file_content = {"stack": stack, "deploy-to": ctx.obj.deployer.name}
|
||||
if default_spec_file_content:
|
||||
spec_file_content.update(default_spec_file_content)
|
||||
config_variables = _parse_config_variables(config)
|
||||
if config_variables:
|
||||
# Implement merge, since update() overwrites
|
||||
orig_config = spec_file_content.get("config", {})
|
||||
new_config = config_variables["config"]
|
||||
merged_config = {**new_config, **orig_config}
|
||||
spec_file_content.update({"config": merged_config})
|
||||
if debug:
|
||||
print(f"Creating spec file for stack: {stack} with content: {spec_file_content}")
|
||||
|
||||
ports = _get_mapped_ports(stack, map_ports_to_host)
|
||||
spec_file_content["ports"] = ports
|
||||
|
||||
named_volumes = _get_named_volumes(stack)
|
||||
if named_volumes:
|
||||
volume_descriptors = {}
|
||||
for named_volume in named_volumes:
|
||||
volume_descriptors[named_volume] = f"./data/{named_volume}"
|
||||
spec_file_content["volumes"] = volume_descriptors
|
||||
|
||||
with open(output, "w") as output_file:
|
||||
yaml.dump(spec_file_content, output_file)
|
||||
|
||||
|
||||
def _write_config_file(spec_file: Path, config_env_file: Path):
|
||||
spec_content = get_parsed_deployment_spec(spec_file)
|
||||
# Note: we want to write an empty file even if we have no config variables
|
||||
with open(config_env_file, "w") as output_file:
|
||||
if "config" in spec_content and spec_content["config"]:
|
||||
config_vars = spec_content["config"]
|
||||
if config_vars:
|
||||
for variable_name, variable_value in config_vars.items():
|
||||
output_file.write(f"{variable_name}={variable_value}\n")
|
||||
|
||||
|
||||
def _copy_files_to_directory(file_paths: List[Path], directory: Path):
|
||||
for path in file_paths:
|
||||
# Using copy to preserve the execute bit
|
||||
copy(path, os.path.join(directory, os.path.basename(path)))
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--spec-file", required=True, help="Spec file to use to create this deployment")
|
||||
@click.option("--deployment-dir", help="Create deployment files in this directory")
|
||||
# TODO: Hack
|
||||
@click.option("--network-dir", help="Network configuration supplied in this directory")
|
||||
@click.option("--initial-peers", help="Initial set of persistent peers")
|
||||
@click.pass_context
|
||||
def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
|
||||
# This function fails with a useful error message if the file doens't exist
|
||||
parsed_spec = get_parsed_deployment_spec(spec_file)
|
||||
stack_name = parsed_spec['stack']
|
||||
stack_file = get_stack_file_path(stack_name)
|
||||
parsed_stack = get_parsed_stack_config(stack_name)
|
||||
if global_options(ctx).debug:
|
||||
print(f"parsed spec: {parsed_spec}")
|
||||
if deployment_dir is None:
|
||||
deployment_dir = _make_default_deployment_dir()
|
||||
if os.path.exists(deployment_dir):
|
||||
print(f"Error: {deployment_dir} already exists")
|
||||
sys.exit(1)
|
||||
os.mkdir(deployment_dir)
|
||||
# Copy spec file and the stack file into the deployment dir
|
||||
copyfile(spec_file, os.path.join(deployment_dir, "spec.yml"))
|
||||
copyfile(stack_file, os.path.join(deployment_dir, os.path.basename(stack_file)))
|
||||
# Copy any config varibles from the spec file into an env file suitable for compose
|
||||
_write_config_file(spec_file, os.path.join(deployment_dir, "config.env"))
|
||||
# Copy the pod files into the deployment dir, fixing up content
|
||||
pods = get_pod_list(parsed_stack)
|
||||
destination_compose_dir = os.path.join(deployment_dir, "compose")
|
||||
os.mkdir(destination_compose_dir)
|
||||
destination_pods_dir = os.path.join(deployment_dir, "pods")
|
||||
os.mkdir(destination_pods_dir)
|
||||
data_dir = Path(__file__).absolute().parent.parent.joinpath("data")
|
||||
yaml = get_yaml()
|
||||
for pod in pods:
|
||||
pod_file_path = get_pod_file_path(parsed_stack, pod)
|
||||
parsed_pod_file = yaml.load(open(pod_file_path, "r"))
|
||||
extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod)
|
||||
destination_pod_dir = os.path.join(destination_pods_dir, pod)
|
||||
os.mkdir(destination_pod_dir)
|
||||
if global_options(ctx).debug:
|
||||
print(f"extra config dirs: {extra_config_dirs}")
|
||||
_fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir)
|
||||
with open(os.path.join(destination_compose_dir, "docker-compose-%s.yml" % pod), "w") as output_file:
|
||||
yaml.dump(parsed_pod_file, output_file)
|
||||
# Copy the config files for the pod, if any
|
||||
config_dirs = {pod}
|
||||
config_dirs = config_dirs.union(extra_config_dirs)
|
||||
for config_dir in config_dirs:
|
||||
source_config_dir = data_dir.joinpath("config", config_dir)
|
||||
if os.path.exists(source_config_dir):
|
||||
destination_config_dir = os.path.join(deployment_dir, "config", config_dir)
|
||||
# If the same config dir appears in multiple pods, it may already have been copied
|
||||
if not os.path.exists(destination_config_dir):
|
||||
copytree(source_config_dir, destination_config_dir)
|
||||
# Copy the script files for the pod, if any
|
||||
if pod_has_scripts(parsed_stack, pod):
|
||||
destination_script_dir = os.path.join(destination_pod_dir, "scripts")
|
||||
os.mkdir(destination_script_dir)
|
||||
script_paths = get_pod_script_paths(parsed_stack, pod)
|
||||
_copy_files_to_directory(script_paths, destination_script_dir)
|
||||
# Delegate to the stack's Python code
|
||||
# The deploy create command doesn't require a --stack argument so we need to insert the
|
||||
# stack member here.
|
||||
deployment_command_context = ctx.obj
|
||||
deployment_command_context.stack = stack_name
|
||||
deployment_context = DeploymentContext(Path(deployment_dir), deployment_command_context)
|
||||
# Call the deployer to generate any deployer-specific files (e.g. for kind)
|
||||
deployer_config_generator = getDeployerConfigGenerator(parsed_spec["deploy-to"])
|
||||
# TODO: make deployment_dir a Path above
|
||||
deployer_config_generator.generate(Path(deployment_dir))
|
||||
call_stack_deploy_create(deployment_context, [network_dir, initial_peers])
|
||||
|
||||
|
||||
# TODO: this code should be in the stack .py files but
|
||||
# we haven't yet figured out how to integrate click across
|
||||
# the plugin boundary
|
||||
@click.command()
|
||||
@click.option("--node-moniker", help="Moniker for this node")
|
||||
@click.option("--chain-id", help="The new chain id")
|
||||
@click.option("--key-name", help="Name for new node key")
|
||||
@click.option("--gentx-files", help="List of comma-delimited gentx filenames from other nodes")
|
||||
@click.option("--genesis-file", help="Genesis file for the network")
|
||||
@click.option("--initialize-network", is_flag=True, default=False, help="Initialize phase")
|
||||
@click.option("--join-network", is_flag=True, default=False, help="Join phase")
|
||||
@click.option("--create-network", is_flag=True, default=False, help="Create phase")
|
||||
@click.option("--network-dir", help="Directory for network files")
|
||||
@click.argument('extra_args', nargs=-1)
|
||||
@click.pass_context
|
||||
def setup(ctx, node_moniker, chain_id, key_name, gentx_files, genesis_file, initialize_network, join_network, create_network,
|
||||
network_dir, extra_args):
|
||||
parmeters = LaconicStackSetupCommand(chain_id, node_moniker, key_name, initialize_network, join_network, create_network,
|
||||
gentx_files, genesis_file, network_dir)
|
||||
call_stack_deploy_setup(ctx.obj, parmeters, extra_args)
|
||||
@@ -0,0 +1,105 @@
|
||||
# Copyright © 2023 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
from kubernetes import client
|
||||
from typing import Any, List, Set
|
||||
|
||||
from stack_orchestrator.opts import opts
|
||||
from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files
|
||||
from stack_orchestrator.deploy.k8s.helpers import parsed_pod_files_map_from_file_names
|
||||
|
||||
|
||||
class ClusterInfo:
|
||||
parsed_pod_yaml_map: Any = {}
|
||||
image_set: Set[str] = set()
|
||||
app_name: str = "test-app"
|
||||
deployment_name: str = "test-deployment"
|
||||
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def int_from_pod_files(self, pod_files: List[str]):
|
||||
self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files)
|
||||
# Find the set of images in the pods
|
||||
for pod_name in self.parsed_pod_yaml_map:
|
||||
pod = self.parsed_pod_yaml_map[pod_name]
|
||||
services = pod["services"]
|
||||
for service_name in services:
|
||||
service_info = services[service_name]
|
||||
image = service_info["image"]
|
||||
self.image_set.add(image)
|
||||
if opts.o.debug:
|
||||
print(f"image_set: {self.image_set}")
|
||||
|
||||
def get_pvcs(self):
|
||||
result = []
|
||||
volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
|
||||
if opts.o.debug:
|
||||
print(f"Volumes: {volumes}")
|
||||
for volume_name in volumes:
|
||||
spec = client.V1PersistentVolumeClaimSpec(
|
||||
storage_class_name="standard",
|
||||
access_modes=["ReadWriteOnce"],
|
||||
resources=client.V1ResourceRequirements(
|
||||
requests={"storage": "2Gi"}
|
||||
)
|
||||
)
|
||||
pvc = client.V1PersistentVolumeClaim(
|
||||
metadata=client.V1ObjectMeta(name=volume_name,
|
||||
labels={"volume-label": volume_name}),
|
||||
spec=spec,
|
||||
)
|
||||
result.append(pvc)
|
||||
return result
|
||||
|
||||
# to suit the deployment, and also annotate the container specs to point at said volumes
|
||||
def get_deployment(self):
|
||||
containers = []
|
||||
for pod_name in self.parsed_pod_yaml_map:
|
||||
pod = self.parsed_pod_yaml_map[pod_name]
|
||||
services = pod["services"]
|
||||
for service_name in services:
|
||||
container_name = service_name
|
||||
service_info = services[service_name]
|
||||
image = service_info["image"]
|
||||
volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name)
|
||||
container = client.V1Container(
|
||||
name=container_name,
|
||||
image=image,
|
||||
ports=[client.V1ContainerPort(container_port=80)],
|
||||
volume_mounts=volume_mounts,
|
||||
resources=client.V1ResourceRequirements(
|
||||
requests={"cpu": "100m", "memory": "200Mi"},
|
||||
limits={"cpu": "500m", "memory": "500Mi"},
|
||||
),
|
||||
)
|
||||
containers.append(container)
|
||||
volumes = volumes_for_pod_files(self.parsed_pod_yaml_map)
|
||||
template = client.V1PodTemplateSpec(
|
||||
metadata=client.V1ObjectMeta(labels={"app": self.app_name}),
|
||||
spec=client.V1PodSpec(containers=containers, volumes=volumes),
|
||||
)
|
||||
spec = client.V1DeploymentSpec(
|
||||
replicas=1, template=template, selector={
|
||||
"matchLabels":
|
||||
{"app": self.app_name}})
|
||||
|
||||
deployment = client.V1Deployment(
|
||||
api_version="apps/v1",
|
||||
kind="Deployment",
|
||||
metadata=client.V1ObjectMeta(name=self.deployment_name),
|
||||
spec=spec,
|
||||
)
|
||||
return deployment
|
||||
@@ -0,0 +1,128 @@
|
||||
# Copyright © 2023 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
from pathlib import Path
|
||||
from kubernetes import client, config
|
||||
|
||||
from stack_orchestrator.deploy.deployer import Deployer, DeployerConfigGenerator
|
||||
from stack_orchestrator.deploy.k8s.helpers import create_cluster, destroy_cluster, load_images_into_kind
|
||||
from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, log_stream_from_string, generate_kind_config
|
||||
from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo
|
||||
from stack_orchestrator.opts import opts
|
||||
|
||||
|
||||
class K8sDeployer(Deployer):
|
||||
name: str = "k8s"
|
||||
core_api: client.CoreV1Api
|
||||
apps_api: client.AppsV1Api
|
||||
k8s_namespace: str = "default"
|
||||
kind_cluster_name: str
|
||||
cluster_info : ClusterInfo
|
||||
|
||||
def __init__(self, compose_files, compose_project_name, compose_env_file) -> None:
|
||||
if (opts.o.debug):
|
||||
print(f"Compose files: {compose_files}")
|
||||
print(f"Project name: {compose_project_name}")
|
||||
print(f"Env file: {compose_env_file}")
|
||||
self.kind_cluster_name = compose_project_name
|
||||
self.cluster_info = ClusterInfo()
|
||||
self.cluster_info.int_from_pod_files(compose_files)
|
||||
|
||||
def connect_api(self):
|
||||
config.load_kube_config(context=f"kind-{self.kind_cluster_name}")
|
||||
self.core_api = client.CoreV1Api()
|
||||
self.apps_api = client.AppsV1Api()
|
||||
|
||||
def up(self, detach, services):
|
||||
# Create the kind cluster
|
||||
# HACK: pass in the config file path here
|
||||
create_cluster(self.kind_cluster_name, "./test-deployment-dir/kind-config.yml")
|
||||
self.connect_api()
|
||||
# Ensure the referenced containers are copied into kind
|
||||
load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set)
|
||||
# Figure out the PVCs for this deployment
|
||||
pvcs = self.cluster_info.get_pvcs()
|
||||
for pvc in pvcs:
|
||||
if opts.o.debug:
|
||||
print(f"Sending this: {pvc}")
|
||||
pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace=self.k8s_namespace)
|
||||
if opts.o.debug:
|
||||
print("PVCs created:")
|
||||
print(f"{pvc_resp}")
|
||||
# Process compose files into a Deployment
|
||||
deployment = self.cluster_info.get_deployment()
|
||||
# Create the k8s objects
|
||||
if opts.o.debug:
|
||||
print(f"Sending this: {deployment}")
|
||||
deployment_resp = self.apps_api.create_namespaced_deployment(
|
||||
body=deployment, namespace=self.k8s_namespace
|
||||
)
|
||||
if opts.o.debug:
|
||||
print("Deployment created:")
|
||||
print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \
|
||||
{deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}")
|
||||
|
||||
def down(self, timeout, volumes):
|
||||
# Delete the k8s objects
|
||||
# Destroy the kind cluster
|
||||
destroy_cluster(self.kind_cluster_name)
|
||||
|
||||
def ps(self):
|
||||
self.connect_api()
|
||||
# Call whatever API we need to get the running container list
|
||||
ret = self.core_api.list_pod_for_all_namespaces(watch=False)
|
||||
if ret.items:
|
||||
for i in ret.items:
|
||||
print("%s\t%s\t%s" % (i.status.pod_ip, i.metadata.namespace, i.metadata.name))
|
||||
ret = self.core_api.list_node(pretty=True, watch=False)
|
||||
return []
|
||||
|
||||
def port(self, service, private_port):
|
||||
# Since we handle the port mapping, need to figure out where this comes from
|
||||
# Also look into whether it makes sense to get ports for k8s
|
||||
pass
|
||||
|
||||
def execute(self, service_name, command, tty, envs):
|
||||
# Call the API to execute a command in a running container
|
||||
pass
|
||||
|
||||
def logs(self, services, tail, follow, stream):
|
||||
self.connect_api()
|
||||
pods = pods_in_deployment(self.core_api, "test-deployment")
|
||||
if len(pods) > 1:
|
||||
print("Warning: more than one pod in the deployment")
|
||||
k8s_pod_name = pods[0]
|
||||
log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
|
||||
return log_stream_from_string(log_data)
|
||||
|
||||
def run(self, image, command, user, volumes, entrypoint=None):
|
||||
# We need to figure out how to do this -- check why we're being called first
|
||||
pass
|
||||
|
||||
|
||||
class K8sDeployerConfigGenerator(DeployerConfigGenerator):
|
||||
config_file_name: str = "kind-config.yml"
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
def generate(self, deployment_dir: Path):
|
||||
# Check the file isn't already there
|
||||
# Get the config file contents
|
||||
content = generate_kind_config(deployment_dir)
|
||||
config_file = deployment_dir.joinpath(self.config_file_name)
|
||||
# Write the file
|
||||
with open(config_file, "w") as output_file:
|
||||
output_file.write(content)
|
||||
@@ -0,0 +1,212 @@
|
||||
# Copyright © 2023 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
from kubernetes import client
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from typing import Any, Set
|
||||
|
||||
from stack_orchestrator.opts import opts
|
||||
from stack_orchestrator.util import get_yaml
|
||||
|
||||
|
||||
def _run_command(command: str):
|
||||
if opts.o.debug:
|
||||
print(f"Running: {command}")
|
||||
result = subprocess.run(command, shell=True)
|
||||
if opts.o.debug:
|
||||
print(f"Result: {result}")
|
||||
|
||||
|
||||
def create_cluster(name: str, config_file: str):
|
||||
_run_command(f"kind create cluster --name {name} --config {config_file}")
|
||||
|
||||
|
||||
def destroy_cluster(name: str):
|
||||
_run_command(f"kind delete cluster --name {name}")
|
||||
|
||||
|
||||
def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]):
|
||||
for image in image_set:
|
||||
_run_command(f"kind load docker-image {image} --name {kind_cluster_name}")
|
||||
|
||||
|
||||
def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
|
||||
pods = []
|
||||
pod_response = core_api.list_namespaced_pod(namespace="default", label_selector="app=test-app")
|
||||
if opts.o.debug:
|
||||
print(f"pod_response: {pod_response}")
|
||||
for pod_info in pod_response.items:
|
||||
pod_name = pod_info.metadata.name
|
||||
pods.append(pod_name)
|
||||
return pods
|
||||
|
||||
|
||||
def log_stream_from_string(s: str):
|
||||
# Note response has to be UTF-8 encoded because the caller expects to decode it
|
||||
yield ("ignore", s.encode())
|
||||
|
||||
|
||||
def named_volumes_from_pod_files(parsed_pod_files):
|
||||
# Parse the compose files looking for named volumes
|
||||
named_volumes = []
|
||||
for pod in parsed_pod_files:
|
||||
parsed_pod_file = parsed_pod_files[pod]
|
||||
if "volumes" in parsed_pod_file:
|
||||
volumes = parsed_pod_file["volumes"]
|
||||
for volume in volumes.keys():
|
||||
# Volume definition looks like:
|
||||
# 'laconicd-data': None
|
||||
named_volumes.append(volume)
|
||||
return named_volumes
|
||||
|
||||
|
||||
def volume_mounts_for_service(parsed_pod_files, service):
|
||||
result = []
|
||||
# Find the service
|
||||
for pod in parsed_pod_files:
|
||||
parsed_pod_file = parsed_pod_files[pod]
|
||||
if "services" in parsed_pod_file:
|
||||
services = parsed_pod_file["services"]
|
||||
for service_name in services:
|
||||
if service_name == service:
|
||||
service_obj = services[service_name]
|
||||
if "volumes" in service_obj:
|
||||
volumes = service_obj["volumes"]
|
||||
for mount_string in volumes:
|
||||
# Looks like: test-data:/data
|
||||
(volume_name, mount_path) = mount_string.split(":")
|
||||
volume_device = client.V1VolumeMount(mount_path=mount_path, name=volume_name)
|
||||
result.append(volume_device)
|
||||
return result
|
||||
|
||||
|
||||
def volumes_for_pod_files(parsed_pod_files):
|
||||
result = []
|
||||
for pod in parsed_pod_files:
|
||||
parsed_pod_file = parsed_pod_files[pod]
|
||||
if "volumes" in parsed_pod_file:
|
||||
volumes = parsed_pod_file["volumes"]
|
||||
for volume_name in volumes.keys():
|
||||
claim = client.V1PersistentVolumeClaimVolumeSource(claim_name=volume_name)
|
||||
volume = client.V1Volume(name=volume_name, persistent_volume_claim=claim)
|
||||
result.append(volume)
|
||||
return result
|
||||
|
||||
|
||||
def _get_host_paths_for_volumes(parsed_pod_files):
|
||||
result = {}
|
||||
for pod in parsed_pod_files:
|
||||
parsed_pod_file = parsed_pod_files[pod]
|
||||
if "volumes" in parsed_pod_file:
|
||||
volumes = parsed_pod_file["volumes"]
|
||||
for volume_name in volumes.keys():
|
||||
volume_definition = volumes[volume_name]
|
||||
host_path = volume_definition["driver_opts"]["device"]
|
||||
result[volume_name] = host_path
|
||||
return result
|
||||
|
||||
|
||||
def parsed_pod_files_map_from_file_names(pod_files):
|
||||
parsed_pod_yaml_map : Any = {}
|
||||
for pod_file in pod_files:
|
||||
with open(pod_file, "r") as pod_file_descriptor:
|
||||
parsed_pod_file = get_yaml().load(pod_file_descriptor)
|
||||
parsed_pod_yaml_map[pod_file] = parsed_pod_file
|
||||
if opts.o.debug:
|
||||
print(f"parsed_pod_yaml_map: {parsed_pod_yaml_map}")
|
||||
return parsed_pod_yaml_map
|
||||
|
||||
|
||||
def _generate_kind_mounts(parsed_pod_files):
|
||||
volume_definitions = []
|
||||
volume_host_path_map = _get_host_paths_for_volumes(parsed_pod_files)
|
||||
for pod in parsed_pod_files:
|
||||
parsed_pod_file = parsed_pod_files[pod]
|
||||
if "services" in parsed_pod_file:
|
||||
services = parsed_pod_file["services"]
|
||||
for service_name in services:
|
||||
service_obj = services[service_name]
|
||||
if "volumes" in service_obj:
|
||||
volumes = service_obj["volumes"]
|
||||
for mount_string in volumes:
|
||||
# Looks like: test-data:/data
|
||||
(volume_name, mount_path) = mount_string.split(":")
|
||||
volume_definitions.append(
|
||||
f" - hostPath: {volume_host_path_map[volume_name]}\n containerPath: /var/local-path-provisioner"
|
||||
)
|
||||
return (
|
||||
"" if len(volume_definitions) == 0 else (
|
||||
" extraMounts:\n"
|
||||
f"{''.join(volume_definitions)}"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _generate_kind_port_mappings(parsed_pod_files):
|
||||
port_definitions = []
|
||||
for pod in parsed_pod_files:
|
||||
parsed_pod_file = parsed_pod_files[pod]
|
||||
if "services" in parsed_pod_file:
|
||||
services = parsed_pod_file["services"]
|
||||
for service_name in services:
|
||||
service_obj = services[service_name]
|
||||
if "ports" in service_obj:
|
||||
ports = service_obj["ports"]
|
||||
for port_string in ports:
|
||||
# TODO handle the complex cases
|
||||
# Looks like: 80 or something more complicated
|
||||
port_definitions.append(f" - containerPort: {port_string}\n hostPort: {port_string}")
|
||||
return (
|
||||
"" if len(port_definitions) == 0 else (
|
||||
" extraPortMappings:\n"
|
||||
f"{''.join(port_definitions)}"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# This needs to know:
|
||||
# The service ports for the cluster
|
||||
# The bind mounted volumes for the cluster
|
||||
#
|
||||
# Make ports like this:
|
||||
# extraPortMappings:
|
||||
# - containerPort: 80
|
||||
# hostPort: 80
|
||||
# # optional: set the bind address on the host
|
||||
# # 0.0.0.0 is the current default
|
||||
# listenAddress: "127.0.0.1"
|
||||
# # optional: set the protocol to one of TCP, UDP, SCTP.
|
||||
# # TCP is the default
|
||||
# protocol: TCP
|
||||
# Make bind mounts like this:
|
||||
# extraMounts:
|
||||
# - hostPath: /path/to/my/files
|
||||
# containerPath: /files
|
||||
def generate_kind_config(deployment_dir: Path):
|
||||
compose_file_dir = deployment_dir.joinpath("compose")
|
||||
# TODO: this should come from the stack file, not this way
|
||||
pod_files = [p for p in compose_file_dir.iterdir() if p.is_file()]
|
||||
parsed_pod_files_map = parsed_pod_files_map_from_file_names(pod_files)
|
||||
port_mappings_yml = _generate_kind_port_mappings(parsed_pod_files_map)
|
||||
mounts_yml = _generate_kind_mounts(parsed_pod_files_map)
|
||||
return (
|
||||
"kind: Cluster\n"
|
||||
"apiVersion: kind.x-k8s.io/v1alpha4\n"
|
||||
"nodes:\n"
|
||||
"- role: control-plane\n"
|
||||
f"{port_mappings_yml}\n"
|
||||
f"{mounts_yml}\n"
|
||||
)
|
||||
@@ -0,0 +1,30 @@
|
||||
# Copyright © 2022, 2023 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
from pathlib import Path
|
||||
import typing
|
||||
from stack_orchestrator.util import get_yaml
|
||||
|
||||
|
||||
class Spec:
|
||||
|
||||
obj: typing.Any
|
||||
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def init_from_file(self, file_path: Path):
|
||||
with file_path:
|
||||
self.obj = get_yaml().load(open(file_path, "r"))
|
||||
@@ -0,0 +1,30 @@
|
||||
# Copyright © 2023 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
from pathlib import Path
|
||||
import typing
|
||||
from stack_orchestrator.util import get_yaml
|
||||
|
||||
|
||||
class Stack:
|
||||
|
||||
obj: typing.Any
|
||||
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def init_from_file(self, file_path: Path):
|
||||
with file_path:
|
||||
self.obj = get_yaml().load(open(file_path, "r"))
|
||||
@@ -0,0 +1,23 @@
|
||||
# Copyright © 2023 Vulcanize
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class State(Enum):
|
||||
CREATED = 1
|
||||
CONFIGURED = 2
|
||||
STARTED = 3
|
||||
STOPPED = 4
|
||||
Reference in New Issue
Block a user