diff --git a/stack_orchestrator/deploy/compose/deploy_docker.py b/stack_orchestrator/deploy/compose/deploy_docker.py index 565fcfa2..d14ee9ca 100644 --- a/stack_orchestrator/deploy/compose/deploy_docker.py +++ b/stack_orchestrator/deploy/compose/deploy_docker.py @@ -94,6 +94,40 @@ class DockerDeployer(Deployer): except DockerException as e: raise DeployerException(e) + def run_job(self, job_name: str, release_name: str = None): + # release_name is ignored for Docker deployments (only used for K8s/Helm) + if not opts.o.dry_run: + try: + # Find job compose file in compose-jobs directory + # The deployment should have compose-jobs/docker-compose-.yml + if not self.docker.compose_files: + raise DeployerException("No compose files configured") + + # Deployment directory is parent of compose directory + compose_dir = Path(self.docker.compose_files[0]).parent + deployment_dir = compose_dir.parent + job_compose_file = deployment_dir / "compose-jobs" / f"docker-compose-{job_name}.yml" + + if not job_compose_file.exists(): + raise DeployerException(f"Job compose file not found: {job_compose_file}") + + if opts.o.verbose: + print(f"Running job from: {job_compose_file}") + + # Create a DockerClient for the job compose file with same project name and env file + # This allows the job to access volumes from the main deployment + job_docker = DockerClient( + compose_files=[job_compose_file], + compose_project_name=self.docker.compose_project_name, + compose_env_file=self.docker.compose_env_file + ) + + # Run the job with --rm flag to remove container after completion + return job_docker.compose.run(service=job_name, remove=True, tty=True) + + except DockerException as e: + raise DeployerException(e) + class DockerDeployerConfigGenerator(DeployerConfigGenerator): diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index f8802758..f68ea796 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -188,6 +188,18 @@ def logs_operation(ctx, tail: int, follow: bool, extra_args: str): print(stream_content.decode("utf-8"), end="") +def run_job_operation(ctx, job_name: str, release_name: str = None): + global_context = ctx.parent.parent.obj + if not global_context.dry_run: + print(f"Running job: {job_name}") + try: + ctx.obj.deployer.run_job(job_name, release_name) + print(f"Job {job_name} completed successfully") + except Exception as e: + print(f"Error running job {job_name}: {e}") + sys.exit(1) + + @command.command() @click.argument('extra_args', nargs=-1) # help: command: up @click.pass_context diff --git a/stack_orchestrator/deploy/deployer.py b/stack_orchestrator/deploy/deployer.py index 15db44c2..766833bf 100644 --- a/stack_orchestrator/deploy/deployer.py +++ b/stack_orchestrator/deploy/deployer.py @@ -55,6 +55,10 @@ class Deployer(ABC): def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False): pass + @abstractmethod + def run_job(self, job_name: str, release_name: str = None): + pass + class DeployerException(Exception): def __init__(self, *args: object) -> None: diff --git a/stack_orchestrator/deploy/deployment.py b/stack_orchestrator/deploy/deployment.py index 7021c733..6b254225 100644 --- a/stack_orchestrator/deploy/deployment.py +++ b/stack_orchestrator/deploy/deployment.py @@ -167,3 +167,14 @@ def status(ctx): def update(ctx): ctx.obj = make_deploy_context(ctx) update_operation(ctx) + + +@command.command() +@click.argument('job_name') +@click.option('--release-name', help='Helm release name (only for k8s helm chart deployments, defaults to chart name)') +@click.pass_context +def run_job(ctx, job_name, release_name): + '''run a one-time job from the stack''' + from stack_orchestrator.deploy.deploy import run_job_operation + ctx.obj = make_deploy_context(ctx) + run_job_operation(ctx, job_name, release_name) diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index 78c19f0d..b08b0c34 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -461,13 +461,6 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, helm stack_name = parsed_spec["stack"] deployment_type = parsed_spec[constants.deploy_to_key] - # Branch to Helm chart generation flow early if --helm-chart flag is set - if deployment_type == "k8s" and helm_chart: - from stack_orchestrator.deploy.k8s.helm.chart_generator import generate_helm_chart - generate_helm_chart(stack_name, spec_file, deployment_dir) - return # Exit early, completely separate from existing k8s deployment flow - - # Existing deployment flow continues unchanged stack_file = get_stack_path(stack_name).joinpath(constants.stack_file_name) parsed_stack = get_parsed_stack_config(stack_name) if opts.o.debug: @@ -482,7 +475,17 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, helm # Copy spec file and the stack file into the deployment dir copyfile(spec_file, deployment_dir_path.joinpath(constants.spec_file_name)) copyfile(stack_file, deployment_dir_path.joinpath(constants.stack_file_name)) + + # Create deployment.yml with cluster-id _create_deployment_file(deployment_dir_path) + + # Branch to Helm chart generation flow if --helm-chart flag is set + if deployment_type == "k8s" and helm_chart: + from stack_orchestrator.deploy.k8s.helm.chart_generator import generate_helm_chart + generate_helm_chart(stack_name, spec_file, deployment_dir_path) + return # Exit early for helm chart generation + + # Existing deployment flow continues unchanged # Copy any config varibles from the spec file into an env file suitable for compose _write_config_file(spec_file, deployment_dir_path.joinpath(constants.config_file_name)) # Copy any k8s config file into the deployment dir diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index b254fd4c..1207260c 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -510,6 +510,26 @@ class K8sDeployer(Deployer): # We need to figure out how to do this -- check why we're being called first pass + def run_job(self, job_name: str, release_name: str = None): + if not opts.o.dry_run: + from stack_orchestrator.deploy.k8s.helm.job_runner import run_helm_job + + # Check if this is a helm-based deployment + chart_dir = self.deployment_dir / "chart" + if not chart_dir.exists(): + # TODO: Implement job support for compose-based K8s deployments + raise Exception(f"Job support is only available for helm-based deployments. Chart directory not found: {chart_dir}") + + # Run the job using the helm job runner + run_helm_job( + chart_dir=chart_dir, + job_name=job_name, + release_name=release_name, + namespace=self.k8s_namespace, + timeout=600, + verbose=opts.o.verbose + ) + def is_kind(self): return self.type == "k8s-kind" diff --git a/stack_orchestrator/deploy/k8s/helm/chart_generator.py b/stack_orchestrator/deploy/k8s/helm/chart_generator.py index fe8d0c49..d2f279cb 100644 --- a/stack_orchestrator/deploy/k8s/helm/chart_generator.py +++ b/stack_orchestrator/deploy/k8s/helm/chart_generator.py @@ -104,34 +104,29 @@ def _post_process_chart(chart_dir: Path, chart_name: str, jobs: list) -> None: _wrap_job_templates_with_conditionals(chart_dir, jobs) -def generate_helm_chart(stack_path: str, spec_file: str, deployment_dir: str = None) -> None: +def generate_helm_chart(stack_path: str, spec_file: str, deployment_dir_path: Path) -> None: """ Generate a self-sufficient Helm chart from stack compose files using Kompose. Args: stack_path: Path to the stack directory spec_file: Path to the deployment spec file - deployment_dir: Optional directory for deployment output + deployment_dir_path: Deployment directory path (already created with deployment.yml) Output structure: deployment-dir/ - ├── spec.yml # Reference - ├── stack.yml # Reference - └── chart/ # Self-sufficient Helm chart + ├── deployment.yml # Contains cluster-id + ├── spec.yml # Reference + ├── stack.yml # Reference + └── chart/ # Self-sufficient Helm chart ├── Chart.yaml ├── README.md └── templates/ └── *.yaml TODO: Enhancements: - - Parse generated templates and extract values to values.yaml - - Replace hardcoded image tags with {{ .Values.image.tag }} - - Replace hardcoded PVC sizes with {{ .Values.persistence.size }} - Convert Deployments to StatefulSets for stateful services (zenithd, postgres) - Add _helpers.tpl with common label/selector functions - - Embed config files (scripts, templates) into ConfigMap templates - - Generate Secret templates for validator keys with placeholders - - Add init containers for genesis/config setup - Enhance Chart.yaml with proper metadata (version, description, etc.) """ @@ -142,35 +137,31 @@ def generate_helm_chart(stack_path: str, spec_file: str, deployment_dir: str = N if not check_kompose_available(): error_exit("kompose not found in PATH.\n") - # 2. Setup deployment directory - if deployment_dir: - deployment_dir_path = Path(deployment_dir) - else: - deployment_dir_path = Path(f"{stack_name}-deployment") + # 2. Read cluster-id from deployment.yml + deployment_file = deployment_dir_path / constants.deployment_file_name + if not deployment_file.exists(): + error_exit(f"Deployment file not found: {deployment_file}") - if deployment_dir_path.exists(): - error_exit(f"Deployment directory already exists: {deployment_dir_path}") + yaml = get_yaml() + deployment_config = yaml.load(open(deployment_file, "r")) + cluster_id = deployment_config.get(constants.cluster_id_key) + if not cluster_id: + error_exit(f"cluster-id not found in {deployment_file}") + + # 3. Derive chart name from stack name + cluster-id suffix + # Sanitize stack name for use in chart name + sanitized_stack_name = stack_name.replace("_", "-").replace(" ", "-") + + # Extract hex suffix from cluster-id (after the prefix) + # cluster-id format: "laconic-" -> extract the hex part + cluster_id_suffix = cluster_id.split("-", 1)[1] if "-" in cluster_id else cluster_id + + # Combine to create human-readable + unique chart name + chart_name = f"{sanitized_stack_name}-{cluster_id_suffix}" if opts.o.debug: - print(f"Creating deployment directory: {deployment_dir_path}") - - deployment_dir_path.mkdir(parents=True) - - # 3. Copy spec and stack files to deployment directory (for reference) - spec_path = Path(spec_file).resolve() - if not spec_path.exists(): - error_exit(f"Spec file not found: {spec_file}") - - stack_file_path = get_stack_path(stack_path).joinpath(constants.stack_file_name) - if not stack_file_path.exists(): - error_exit(f"Stack file not found: {stack_file_path}") - - shutil.copy(spec_path, deployment_dir_path / constants.spec_file_name) - shutil.copy(stack_file_path, deployment_dir_path / constants.stack_file_name) - - if opts.o.debug: - print(f"Copied spec file: {spec_path}") - print(f"Copied stack file: {stack_file_path}") + print(f"Cluster ID: {cluster_id}") + print(f"Chart name: {chart_name}") # 4. Get compose files from stack (pods + jobs) pods = get_pod_list(parsed_stack) @@ -179,9 +170,6 @@ def generate_helm_chart(stack_path: str, spec_file: str, deployment_dir: str = N jobs = get_job_list(parsed_stack) - # Get clean stack name from stack.yml - chart_name = stack_name.replace("_", "-").replace(" ", "-") - if opts.o.debug: print(f"Found {len(pods)} pod(s) in stack: {pods}") if jobs: @@ -249,6 +237,9 @@ Generated by laconic-so from stack: `{stack_path}` # Install the chart helm install {chart_name} {chart_dir} +# Alternatively, install with your own release name +# helm install {chart_dir} + # Check deployment status kubectl get pods ``` @@ -301,9 +292,10 @@ Edit the generated template files in `templates/` to customize: print("\nDeployment directory structure:") print(f" {deployment_dir_path}/") - print(" ├── spec.yml (reference)") - print(" ├── stack.yml (reference)") - print(" └── chart/ (self-sufficient Helm chart)") + print(" ├── deployment.yml (cluster-id)") + print(" ├── spec.yml (reference)") + print(" ├── stack.yml (reference)") + print(" └── chart/ (self-sufficient Helm chart)") print("\nNext steps:") print(" 1. Review the chart:") @@ -313,9 +305,12 @@ Edit the generated template files in `templates/` to customize: print(" 2. Review generated templates:") print(" ls templates/") print("") - print(" 3. Install to Kubernetes:") + print(f" 3. Install to Kubernetes:") print(f" helm install {chart_name} {chart_dir}") print("") + print(f" # Or use your own release name") + print(f" helm install {chart_dir}") + print("") print(" 4. Check deployment:") print(" kubectl get pods") print("")