From 641052558a99ad74e9da5a129b65ab3bedcf97d3 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Fri, 27 Feb 2026 09:25:57 +0000
Subject: [PATCH 01/26] feat: add secrets support for k8s deployments

Adds a `secrets:` key to spec.yml that references pre-existing k8s
Secrets by name. SO mounts them as envFrom.secretRef on all pod
containers. Secret contents are managed out-of-band by the operator.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 stack_orchestrator/constants.py                |  1 +
 stack_orchestrator/deploy/deployment_create.py |  3 +++
 stack_orchestrator/deploy/k8s/cluster_info.py  | 10 ++++++++++
 stack_orchestrator/deploy/spec.py              |  3 +++
 4 files changed, 17 insertions(+)

diff --git a/stack_orchestrator/constants.py b/stack_orchestrator/constants.py
index 75bd0ebc..5e7b59bf 100644
--- a/stack_orchestrator/constants.py
+++ b/stack_orchestrator/constants.py
@@ -29,6 +29,7 @@ network_key = "network"
 http_proxy_key = "http-proxy"
 image_registry_key = "image-registry"
 configmaps_key = "configmaps"
+secrets_key = "secrets"
 resources_key = "resources"
 volumes_key = "volumes"
 security_key = "security"
diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py
index 511445be..ffbc2872 100644
--- a/stack_orchestrator/deploy/deployment_create.py
+++ b/stack_orchestrator/deploy/deployment_create.py
@@ -477,6 +477,9 @@ def init_operation(
             spec_file_content["volumes"] = {**volume_descriptors, **orig_volumes}
         if configmap_descriptors:
             spec_file_content["configmaps"] = configmap_descriptors
+        if "k8s" in deployer_type:
+            if "secrets" not in spec_file_content:
+                spec_file_content["secrets"] = {}
 
     if opts.o.debug:
         print(
diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py
index 2ebf96f2..088292ca 100644
--- a/stack_orchestrator/deploy/k8s/cluster_info.py
+++ b/stack_orchestrator/deploy/k8s/cluster_info.py
@@ -513,6 +513,16 @@ class ClusterInfo:
                         )
                     )
                 ]
+                # Mount user-declared secrets from spec.yml
+                for user_secret_name in self.spec.get_secrets():
+                    env_from.append(
+                        client.V1EnvFromSource(
+                            secret_ref=client.V1SecretEnvSource(
+                                name=user_secret_name,
+                                optional=True,
+                            )
+                        )
+                    )
                 container_resources = self._resolve_container_resources(
                     container_name, service_info, global_resources
                 )
diff --git a/stack_orchestrator/deploy/spec.py b/stack_orchestrator/deploy/spec.py
index bd62779e..e77b9581 100644
--- a/stack_orchestrator/deploy/spec.py
+++ b/stack_orchestrator/deploy/spec.py
@@ -115,6 +115,9 @@ class Spec:
     def get_configmaps(self):
         return self.obj.get(constants.configmaps_key, {})
 
+    def get_secrets(self):
+        return self.obj.get(constants.secrets_key, {})
+
     def get_container_resources(self):
         return Resources(
             self.obj.get(constants.resources_key, {}).get("containers", {})
-- 
2.45.2


From 589ed3cf695915dd2e542432782a322389a5ee87 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Mon, 2 Mar 2026 09:40:25 +0000
Subject: [PATCH 02/26] docs: update CLI reference to match actual code

cli.md:
- Document `start`/`stop` as preferred commands (`up`/`down` as legacy)
- Add --skip-cluster-management flag for start and stop
- Add --delete-volumes flag for stop
- Add missing subcommands: restart, exec, status, port, push-images, run-job
- Add --helm-chart option to deploy create
- Reorganize deploy vs deployment sections for clarity

deployment_patterns.md:
- Add missing --stack flag to deploy create example

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 docs/cli.md                 | 80 +++++++++++++++++++++++++++++++------
 docs/deployment_patterns.md |  2 +-
 2 files changed, 69 insertions(+), 13 deletions(-)

diff --git a/docs/cli.md b/docs/cli.md
index 92cf776a..a871ee50 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -68,7 +68,7 @@ $ laconic-so build-npms --include <package-name> --force-rebuild
 
 ## deploy
 
-The `deploy` command group manages persistent deployments. The general workflow is `deploy init` to generate a spec file, then `deploy create` to create a deployment directory from the spec, then runtime commands like `deploy up` and `deploy down`.
+The `deploy` command group manages persistent deployments. The general workflow is `deploy init` to generate a spec file, then `deploy create` to create a deployment directory from the spec, then runtime commands like `deployment start` and `deployment stop`.
 
 ### deploy init
 
@@ -101,35 +101,91 @@ Options:
 - `--spec-file` (required): spec file to use
 - `--deployment-dir`: target directory for deployment files
 - `--update`: update an existing deployment directory, preserving data volumes and env file. Changed files are backed up with a `.bak` suffix. The deployment's `config.env` and `deployment.yml` are also preserved.
+- `--helm-chart`: generate Helm chart instead of deploying (k8s only)
 - `--network-dir`: network configuration supplied in this directory
 - `--initial-peers`: initial set of persistent peers
 
-### deploy up
+## deployment
 
-Start a deployment:
+Runtime commands for managing a created deployment. Use `--dir` to specify the deployment directory.
+
+### deployment start
+
+Start a deployment (`up` is a legacy alias):
 ```
-$ laconic-so deployment --dir <deployment-dir> up
+$ laconic-so deployment --dir <deployment-dir> start
 ```
 
-### deploy down
+Options:
+- `--stay-attached` / `--detatch-terminal`: attach to container stdout (default: detach)
+- `--skip-cluster-management` / `--perform-cluster-management`: skip kind cluster creation/teardown (default: perform management). Only affects k8s-kind deployments. Use this when multiple stacks share a single cluster.
 
-Stop a deployment:
-```
-$ laconic-so deployment --dir <deployment-dir> down
-```
-Use `--delete-volumes` to also remove data volumes.
+### deployment stop
 
-### deploy ps
+Stop a deployment (`down` is a legacy alias):
+```
+$ laconic-so deployment --dir <deployment-dir> stop
+```
+
+Options:
+- `--delete-volumes` / `--preserve-volumes`: delete data volumes on stop (default: preserve)
+- `--skip-cluster-management` / `--perform-cluster-management`: skip kind cluster teardown (default: perform management). Use this to stop a single deployment without destroying a shared cluster.
+
+### deployment restart
+
+Restart a deployment with GitOps-aware workflow. Pulls latest stack code, syncs the deployment directory from the git-tracked spec, and restarts services:
+```
+$ laconic-so deployment --dir <deployment-dir> restart
+```
+
+See [deployment_patterns.md](deployment_patterns.md) for the recommended GitOps workflow.
+
+### deployment ps
 
 Show running services:
 ```
 $ laconic-so deployment --dir <deployment-dir> ps
 ```
 
-### deploy logs
+### deployment logs
 
 View service logs:
 ```
 $ laconic-so deployment --dir <deployment-dir> logs
 ```
 Use `-f` to follow and `-n <count>` to tail.
+
+### deployment exec
+
+Execute a command in a running service container:
+```
+$ laconic-so deployment --dir <deployment-dir> exec <service-name> "<command>"
+```
+
+### deployment status
+
+Show deployment status:
+```
+$ laconic-so deployment --dir <deployment-dir> status
+```
+
+### deployment port
+
+Show mapped ports for a service:
+```
+$ laconic-so deployment --dir <deployment-dir> port <service-name> <port>
+```
+
+### deployment push-images
+
+Push deployment images to a registry:
+```
+$ laconic-so deployment --dir <deployment-dir> push-images
+```
+
+### deployment run-job
+
+Run a one-time job in the deployment:
+```
+$ laconic-so deployment --dir <deployment-dir> run-job <job-name>
+```
diff --git a/docs/deployment_patterns.md b/docs/deployment_patterns.md
index fdb930d8..9fd7ed0b 100644
--- a/docs/deployment_patterns.md
+++ b/docs/deployment_patterns.md
@@ -30,7 +30,7 @@ git commit -m "Add my-stack deployment configuration"
 git push
 
 # On deployment server: deploy from git-tracked spec
-laconic-so deploy create \
+laconic-so --stack my-stack deploy create \
   --spec-file /path/to/operator-repo/spec.yml \
   --deployment-dir my-deployment
 
-- 
2.45.2


From 74deb3f8d64d4fa44fe0151bea69857efe0edce1 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 3 Mar 2026 13:42:04 +0000
Subject: [PATCH 03/26] feat(k8s): add Job support for non-Helm k8s-kind
 deployments

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 stack_orchestrator/deploy/deploy.py           |  21 +++-
 stack_orchestrator/deploy/deploy_types.py     |   1 +
 stack_orchestrator/deploy/deployer_factory.py |   8 +-
 .../deploy/deployment_create.py               |   4 +-
 stack_orchestrator/deploy/k8s/cluster_info.py | 105 ++++++++++++++++--
 stack_orchestrator/deploy/k8s/deploy_k8s.py   |  79 ++++++++++---
 stack_orchestrator/util.py                    |  16 ++-
 7 files changed, 200 insertions(+), 34 deletions(-)

diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py
index 86c1856c..30f88fa2 100644
--- a/stack_orchestrator/deploy/deploy.py
+++ b/stack_orchestrator/deploy/deploy.py
@@ -35,6 +35,7 @@ from stack_orchestrator.util import (
     get_dev_root_path,
     stack_is_in_deployment,
     resolve_compose_file,
+    get_job_list,
 )
 from stack_orchestrator.deploy.deployer import DeployerException
 from stack_orchestrator.deploy.deployer_factory import getDeployer
@@ -130,6 +131,7 @@ def create_deploy_context(
         compose_files=cluster_context.compose_files,
         compose_project_name=cluster_context.cluster,
         compose_env_file=cluster_context.env_file,
+        job_compose_files=cluster_context.job_compose_files,
     )
     return DeployCommandContext(stack, cluster_context, deployer)
 
@@ -403,7 +405,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
         stack_config = get_parsed_stack_config(stack)
         if stack_config is not None:
             # TODO: syntax check the input here
-            pods_in_scope = stack_config["pods"]
+            pods_in_scope = stack_config.get("pods") or []
             cluster_config = (
                 stack_config["config"] if "config" in stack_config else None
             )
@@ -477,6 +479,22 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
     if ctx.verbose:
         print(f"files: {compose_files}")
 
+    # Gather job compose files (from compose-jobs/ directory in deployment)
+    job_compose_files = []
+    if deployment and stack:
+        stack_config = get_parsed_stack_config(stack)
+        if stack_config:
+            jobs = get_job_list(stack_config)
+            compose_jobs_dir = stack.joinpath("compose-jobs")
+            for job in jobs:
+                job_file_name = os.path.join(
+                    compose_jobs_dir, f"docker-compose-{job}.yml"
+                )
+                if os.path.exists(job_file_name):
+                    job_compose_files.append(job_file_name)
+            if ctx.verbose:
+                print(f"job files: {job_compose_files}")
+
     return ClusterContext(
         ctx,
         cluster,
@@ -485,6 +503,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
         post_start_commands,
         cluster_config,
         env_file,
+        job_compose_files=job_compose_files if job_compose_files else None,
     )
 
 
diff --git a/stack_orchestrator/deploy/deploy_types.py b/stack_orchestrator/deploy/deploy_types.py
index 202e0fa5..68a5f903 100644
--- a/stack_orchestrator/deploy/deploy_types.py
+++ b/stack_orchestrator/deploy/deploy_types.py
@@ -29,6 +29,7 @@ class ClusterContext:
     post_start_commands: List[str]
     config: Optional[str]
     env_file: Optional[str]
+    job_compose_files: Optional[List[str]] = None
 
 
 @dataclass
diff --git a/stack_orchestrator/deploy/deployer_factory.py b/stack_orchestrator/deploy/deployer_factory.py
index 1de14cc5..3bbae74c 100644
--- a/stack_orchestrator/deploy/deployer_factory.py
+++ b/stack_orchestrator/deploy/deployer_factory.py
@@ -34,7 +34,12 @@ def getDeployerConfigGenerator(type: str, deployment_context):
 
 
 def getDeployer(
-    type: str, deployment_context, compose_files, compose_project_name, compose_env_file
+    type: str,
+    deployment_context,
+    compose_files,
+    compose_project_name,
+    compose_env_file,
+    job_compose_files=None,
 ):
     if type == "compose" or type is None:
         return DockerDeployer(
@@ -54,6 +59,7 @@ def getDeployer(
             compose_files,
             compose_project_name,
             compose_env_file,
+            job_compose_files=job_compose_files,
         )
     else:
         print(f"ERROR: deploy-to {type} is not valid")
diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py
index ffbc2872..0546f370 100644
--- a/stack_orchestrator/deploy/deployment_create.py
+++ b/stack_orchestrator/deploy/deployment_create.py
@@ -1017,9 +1017,9 @@ def _write_deployment_files(
                             dirs_exist_ok=True,
                         )
 
-    # Copy the job files into the target dir (for Docker deployments)
+    # Copy the job files into the target dir
     jobs = get_job_list(parsed_stack)
-    if jobs and not parsed_spec.is_kubernetes_deployment():
+    if jobs:
         destination_compose_jobs_dir = target_dir.joinpath("compose-jobs")
         os.makedirs(destination_compose_jobs_dir, exist_ok=True)
         for job in jobs:
diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py
index 088292ca..fe816cb5 100644
--- a/stack_orchestrator/deploy/k8s/cluster_info.py
+++ b/stack_orchestrator/deploy/k8s/cluster_info.py
@@ -72,13 +72,14 @@ def to_k8s_resource_requirements(resources: Resources) -> client.V1ResourceRequi
 
 class ClusterInfo:
     parsed_pod_yaml_map: Any
+    parsed_job_yaml_map: Any
     image_set: Set[str] = set()
     app_name: str
     environment_variables: DeployEnvVars
     spec: Spec
 
     def __init__(self) -> None:
-        pass
+        self.parsed_job_yaml_map = {}
 
     def int(self, pod_files: List[str], compose_env_file, deployment_name, spec: Spec):
         self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files)
@@ -94,6 +95,12 @@ class ClusterInfo:
         if opts.o.debug:
             print(f"Env vars: {self.environment_variables.map}")
 
+    def init_jobs(self, job_files: List[str]):
+        """Initialize parsed job YAML map from job compose files."""
+        self.parsed_job_yaml_map = parsed_pod_files_map_from_file_names(job_files)
+        if opts.o.debug:
+            print(f"Parsed job yaml map: {self.parsed_job_yaml_map}")
+
     def get_nodeports(self):
         nodeports = []
         for pod_name in self.parsed_pod_yaml_map:
@@ -424,15 +431,25 @@ class ClusterInfo:
         # 3. Fall back to spec.yml global (already resolved with DEFAULT fallback)
         return global_resources
 
-    # TODO: put things like image pull policy into an object-scope struct
-    def get_deployment(self, image_pull_policy: Optional[str] = None):
+    def _build_containers(
+        self,
+        parsed_yaml_map: Any,
+        image_pull_policy: Optional[str] = None,
+    ) -> tuple:
+        """Build k8s container specs from parsed compose YAML.
+
+        Returns a tuple of (containers, services, volumes) where:
+        - containers: list of V1Container objects
+        - services: the last services dict processed (used for annotations/labels)
+        - volumes: list of V1Volume objects
+        """
         containers = []
         services = {}
         global_resources = self.spec.get_container_resources()
         if not global_resources:
             global_resources = DEFAULT_CONTAINER_RESOURCES
-        for pod_name in self.parsed_pod_yaml_map:
-            pod = self.parsed_pod_yaml_map[pod_name]
+        for pod_name in parsed_yaml_map:
+            pod = parsed_yaml_map[pod_name]
             services = pod["services"]
             for service_name in services:
                 container_name = service_name
@@ -489,7 +506,7 @@ class ClusterInfo:
                     else image
                 )
                 volume_mounts = volume_mounts_for_service(
-                    self.parsed_pod_yaml_map, service_name
+                    parsed_yaml_map, service_name
                 )
                 # Handle command/entrypoint from compose file
                 # In docker-compose: entrypoint -> k8s command, command -> k8s args
@@ -548,7 +565,14 @@ class ClusterInfo:
                 )
                 containers.append(container)
         volumes = volumes_for_pod_files(
-            self.parsed_pod_yaml_map, self.spec, self.app_name
+            parsed_yaml_map, self.spec, self.app_name
+        )
+        return containers, services, volumes
+
+    # TODO: put things like image pull policy into an object-scope struct
+    def get_deployment(self, image_pull_policy: Optional[str] = None):
+        containers, services, volumes = self._build_containers(
+            self.parsed_pod_yaml_map, image_pull_policy
         )
         registry_config = self.spec.get_image_registry_config()
         if registry_config:
@@ -638,3 +662,70 @@ class ClusterInfo:
             spec=spec,
         )
         return deployment
+
+    def get_jobs(self, image_pull_policy: Optional[str] = None) -> List[client.V1Job]:
+        """Build k8s Job objects from parsed job compose files.
+
+        Each job compose file produces a V1Job with:
+        - restartPolicy: Never
+        - backoffLimit: 0
+        - Name: {app_name}-job-{job_name}
+        """
+        if not self.parsed_job_yaml_map:
+            return []
+
+        jobs = []
+        registry_config = self.spec.get_image_registry_config()
+        if registry_config:
+            secret_name = f"{self.app_name}-registry"
+            image_pull_secrets = [client.V1LocalObjectReference(name=secret_name)]
+        else:
+            image_pull_secrets = []
+
+        for job_file in self.parsed_job_yaml_map:
+            # Build containers for this single job file
+            single_job_map = {job_file: self.parsed_job_yaml_map[job_file]}
+            containers, _services, volumes = self._build_containers(
+                single_job_map, image_pull_policy
+            )
+
+            # Derive job name from file path: docker-compose-<name>.yml -> <name>
+            import os
+
+            base = os.path.basename(job_file)
+            # Strip docker-compose- prefix and .yml suffix
+            job_name = base
+            if job_name.startswith("docker-compose-"):
+                job_name = job_name[len("docker-compose-"):]
+            if job_name.endswith(".yml"):
+                job_name = job_name[: -len(".yml")]
+            elif job_name.endswith(".yaml"):
+                job_name = job_name[: -len(".yaml")]
+
+            template = client.V1PodTemplateSpec(
+                metadata=client.V1ObjectMeta(
+                    labels={"app": self.app_name, "job-name": job_name}
+                ),
+                spec=client.V1PodSpec(
+                    containers=containers,
+                    image_pull_secrets=image_pull_secrets,
+                    volumes=volumes,
+                    restart_policy="Never",
+                ),
+            )
+            job_spec = client.V1JobSpec(
+                template=template,
+                backoff_limit=0,
+            )
+            job = client.V1Job(
+                api_version="batch/v1",
+                kind="Job",
+                metadata=client.V1ObjectMeta(
+                    name=f"{self.app_name}-job-{job_name}",
+                    labels={"app": self.app_name},
+                ),
+                spec=job_spec,
+            )
+            jobs.append(job)
+
+        return jobs
diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py
index f7f8ad43..915b2cd7 100644
--- a/stack_orchestrator/deploy/k8s/deploy_k8s.py
+++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py
@@ -95,6 +95,7 @@ class K8sDeployer(Deployer):
     type: str
     core_api: client.CoreV1Api
     apps_api: client.AppsV1Api
+    batch_api: client.BatchV1Api
     networking_api: client.NetworkingV1Api
     k8s_namespace: str
     kind_cluster_name: str
@@ -110,6 +111,7 @@ class K8sDeployer(Deployer):
         compose_files,
         compose_project_name,
         compose_env_file,
+        job_compose_files=None,
     ) -> None:
         self.type = type
         self.skip_cluster_management = False
@@ -130,9 +132,13 @@ class K8sDeployer(Deployer):
             compose_project_name,
             deployment_context.spec,
         )
+        # Initialize job compose files if provided
+        if job_compose_files:
+            self.cluster_info.init_jobs(job_compose_files)
         if opts.o.debug:
             print(f"Deployment dir: {deployment_context.deployment_dir}")
             print(f"Compose files: {compose_files}")
+            print(f"Job compose files: {job_compose_files}")
             print(f"Project name: {compose_project_name}")
             print(f"Env file: {compose_env_file}")
             print(f"Type: {type}")
@@ -150,6 +156,7 @@ class K8sDeployer(Deployer):
         self.core_api = client.CoreV1Api()
         self.networking_api = client.NetworkingV1Api()
         self.apps_api = client.AppsV1Api()
+        self.batch_api = client.BatchV1Api()
         self.custom_obj_api = client.CustomObjectsApi()
 
     def _ensure_namespace(self):
@@ -293,6 +300,26 @@ class K8sDeployer(Deployer):
                 print("Service created:")
                 print(f"{service_resp}")
 
+    def _create_jobs(self):
+        # Process job compose files into k8s Jobs
+        jobs = self.cluster_info.get_jobs(
+            image_pull_policy=None if self.is_kind() else "Always"
+        )
+        for job in jobs:
+            if opts.o.debug:
+                print(f"Sending this job: {job}")
+            if not opts.o.dry_run:
+                job_resp = self.batch_api.create_namespaced_job(
+                    body=job, namespace=self.k8s_namespace
+                )
+                if opts.o.debug:
+                    print("Job created:")
+                    if job_resp.metadata:
+                        print(
+                            f"  {job_resp.metadata.namespace} "
+                            f"{job_resp.metadata.name}"
+                        )
+
     def _find_certificate_for_host_name(self, host_name):
         all_certificates = self.custom_obj_api.list_namespaced_custom_object(
             group="cert-manager.io",
@@ -384,6 +411,7 @@ class K8sDeployer(Deployer):
 
         self._create_volume_data()
         self._create_deployment()
+        self._create_jobs()
 
         http_proxy_info = self.cluster_info.spec.get_http_proxy()
         # Note: we don't support tls for kind (enabling tls causes errors)
@@ -659,26 +687,43 @@ class K8sDeployer(Deployer):
 
     def run_job(self, job_name: str, helm_release: Optional[str] = None):
         if not opts.o.dry_run:
-            from stack_orchestrator.deploy.k8s.helm.job_runner import run_helm_job
-
             # Check if this is a helm-based deployment
             chart_dir = self.deployment_dir / "chart"
-            if not chart_dir.exists():
-                # TODO: Implement job support for compose-based K8s deployments
-                raise Exception(
-                    f"Job support is only available for helm-based "
-                    f"deployments. Chart directory not found: {chart_dir}"
-                )
+            if chart_dir.exists():
+                from stack_orchestrator.deploy.k8s.helm.job_runner import run_helm_job
 
-            # Run the job using the helm job runner
-            run_helm_job(
-                chart_dir=chart_dir,
-                job_name=job_name,
-                release=helm_release,
-                namespace=self.k8s_namespace,
-                timeout=600,
-                verbose=opts.o.verbose,
-            )
+                # Run the job using the helm job runner
+                run_helm_job(
+                    chart_dir=chart_dir,
+                    job_name=job_name,
+                    release=helm_release,
+                    namespace=self.k8s_namespace,
+                    timeout=600,
+                    verbose=opts.o.verbose,
+                )
+            else:
+                # Non-Helm path: create job from ClusterInfo
+                self.connect_api()
+                jobs = self.cluster_info.get_jobs(
+                    image_pull_policy=None if self.is_kind() else "Always"
+                )
+                # Find the matching job by name
+                target_name = f"{self.cluster_info.app_name}-job-{job_name}"
+                matched_job = None
+                for job in jobs:
+                    if job.metadata and job.metadata.name == target_name:
+                        matched_job = job
+                        break
+                if matched_job is None:
+                    raise Exception(
+                        f"Job '{job_name}' not found. Available jobs: "
+                        f"{[j.metadata.name for j in jobs if j.metadata]}"
+                    )
+                if opts.o.debug:
+                    print(f"Creating job: {target_name}")
+                self.batch_api.create_namespaced_job(
+                    body=matched_job, namespace=self.k8s_namespace
+                )
 
     def is_kind(self):
         return self.type == "k8s-kind"
diff --git a/stack_orchestrator/util.py b/stack_orchestrator/util.py
index fc8437ca..7e1e442c 100644
--- a/stack_orchestrator/util.py
+++ b/stack_orchestrator/util.py
@@ -75,6 +75,8 @@ def get_parsed_stack_config(stack):
 
 def get_pod_list(parsed_stack):
     # Handle both old and new format
+    if "pods" not in parsed_stack or not parsed_stack["pods"]:
+        return []
     pods = parsed_stack["pods"]
     if type(pods[0]) is str:
         result = pods
@@ -103,7 +105,7 @@ def get_job_list(parsed_stack):
 
 def get_plugin_code_paths(stack) -> List[Path]:
     parsed_stack = get_parsed_stack_config(stack)
-    pods = parsed_stack["pods"]
+    pods = parsed_stack.get("pods") or []
     result: Set[Path] = set()
     for pod in pods:
         if type(pod) is str:
@@ -160,8 +162,10 @@ def resolve_job_compose_file(stack, job_name: str):
 
 
 def get_pod_file_path(stack, parsed_stack, pod_name: str):
-    pods = parsed_stack["pods"]
+    pods = parsed_stack.get("pods") or []
     result = None
+    if not pods:
+        return result
     if type(pods[0]) is str:
         result = resolve_compose_file(stack, pod_name)
     else:
@@ -189,9 +193,9 @@ def get_job_file_path(stack, parsed_stack, job_name: str):
 
 
 def get_pod_script_paths(parsed_stack, pod_name: str):
-    pods = parsed_stack["pods"]
+    pods = parsed_stack.get("pods") or []
     result = []
-    if not type(pods[0]) is str:
+    if not pods or not type(pods[0]) is str:
         for pod in pods:
             if pod["name"] == pod_name:
                 pod_root_dir = os.path.join(
@@ -207,9 +211,9 @@ def get_pod_script_paths(parsed_stack, pod_name: str):
 
 
 def pod_has_scripts(parsed_stack, pod_name: str):
-    pods = parsed_stack["pods"]
+    pods = parsed_stack.get("pods") or []
     result = False
-    if type(pods[0]) is str:
+    if not pods or type(pods[0]) is str:
         result = False
     else:
         for pod in pods:
-- 
2.45.2


From e0a84773264fae3dfce65d17f57af1b7598506c7 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 3 Mar 2026 14:02:59 +0000
Subject: [PATCH 04/26] fix(k8s): skip Deployment creation for jobs-only stacks

When a stack defines only jobs: (no pods:), the parsed_pod_yaml_map
is empty. Creating a Deployment with no containers causes a 422 error
from the k8s API. Skip Deployment creation when there are no pods.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 stack_orchestrator/deploy/k8s/deploy_k8s.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py
index 915b2cd7..90ad7655 100644
--- a/stack_orchestrator/deploy/k8s/deploy_k8s.py
+++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py
@@ -263,6 +263,11 @@ class K8sDeployer(Deployer):
                     print(f"{cfg_rsp}")
 
     def _create_deployment(self):
+        # Skip if there are no pods to deploy (e.g. jobs-only stacks)
+        if not self.cluster_info.parsed_pod_yaml_map:
+            if opts.o.debug:
+                print("No pods defined, skipping Deployment creation")
+            return
         # Process compose files into a Deployment
         deployment = self.cluster_info.get_deployment(
             image_pull_policy=None if self.is_kind() else "Always"
-- 
2.45.2


From 9b304b8990e9258beba443a9b801f5046b05ca91 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 3 Mar 2026 14:11:05 +0000
Subject: [PATCH 05/26] fix(k8s): remove job-name label that conflicts with k8s
 auto-label

Kubernetes automatically adds a job-name label to Job pod templates
matching the full Job name. Our custom job-name label used the short
name, causing a 422 validation error. Let k8s manage this label.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 stack_orchestrator/deploy/k8s/cluster_info.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py
index fe816cb5..14ec4c87 100644
--- a/stack_orchestrator/deploy/k8s/cluster_info.py
+++ b/stack_orchestrator/deploy/k8s/cluster_info.py
@@ -704,7 +704,7 @@ class ClusterInfo:
 
             template = client.V1PodTemplateSpec(
                 metadata=client.V1ObjectMeta(
-                    labels={"app": self.app_name, "job-name": job_name}
+                    labels={"app": self.app_name}
                 ),
                 spec=client.V1PodSpec(
                     containers=containers,
-- 
2.45.2


From 47f3068e70d627c19a4f5be2470eca3d437dec37 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 3 Mar 2026 14:13:46 +0000
Subject: [PATCH 06/26] fix(k8s): include job volumes in PVC/ConfigMap/PV
 creation

For jobs-only stacks, named_volumes_from_pod_files() returned empty
because it only scanned parsed_pod_yaml_map. This caused ConfigMaps
and PVCs declared in the spec to be silently skipped.

- Add _all_named_volumes() helper that scans both pod and job maps
- Guard update() against empty parsed_pod_yaml_map (uncaught 404)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 stack_orchestrator/deploy/k8s/cluster_info.py | 12 +++++++++---
 stack_orchestrator/deploy/k8s/deploy_k8s.py   |  4 ++++
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py
index 14ec4c87..be00142f 100644
--- a/stack_orchestrator/deploy/k8s/cluster_info.py
+++ b/stack_orchestrator/deploy/k8s/cluster_info.py
@@ -101,6 +101,12 @@ class ClusterInfo:
         if opts.o.debug:
             print(f"Parsed job yaml map: {self.parsed_job_yaml_map}")
 
+    def _all_named_volumes(self) -> list:
+        """Return named volumes from both pod and job compose files."""
+        volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        volumes.extend(named_volumes_from_pod_files(self.parsed_job_yaml_map))
+        return volumes
+
     def get_nodeports(self):
         nodeports = []
         for pod_name in self.parsed_pod_yaml_map:
@@ -264,7 +270,7 @@ class ClusterInfo:
     def get_pvcs(self):
         result = []
         spec_volumes = self.spec.get_volumes()
-        named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        named_volumes = self._all_named_volumes()
         resources = self.spec.get_volume_resources()
         if not resources:
             resources = DEFAULT_VOLUME_RESOURCES
@@ -308,7 +314,7 @@ class ClusterInfo:
     def get_configmaps(self):
         result = []
         spec_configmaps = self.spec.get_configmaps()
-        named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        named_volumes = self._all_named_volumes()
         for cfg_map_name, cfg_map_path in spec_configmaps.items():
             if cfg_map_name not in named_volumes:
                 if opts.o.debug:
@@ -344,7 +350,7 @@ class ClusterInfo:
     def get_pvs(self):
         result = []
         spec_volumes = self.spec.get_volumes()
-        named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        named_volumes = self._all_named_volumes()
         resources = self.spec.get_volume_resources()
         if not resources:
             resources = DEFAULT_VOLUME_RESOURCES
diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py
index 90ad7655..55513d6e 100644
--- a/stack_orchestrator/deploy/k8s/deploy_k8s.py
+++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py
@@ -632,6 +632,10 @@ class K8sDeployer(Deployer):
         return log_stream_from_string(log_data)
 
     def update(self):
+        if not self.cluster_info.parsed_pod_yaml_map:
+            if opts.o.debug:
+                print("No pods defined, skipping update")
+            return
         self.connect_api()
         ref_deployment = self.cluster_info.get_deployment()
         if not ref_deployment or not ref_deployment.metadata:
-- 
2.45.2


From 8d65cb13a00f10c5d3800d9143235c2e5323fdb1 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 3 Mar 2026 14:26:20 +0000
Subject: [PATCH 07/26] fix(k8s): copy configmap dirs for jobs-only stacks
 during deploy create

The k8s configmap directory copying was inside the `for pod in pods:`
loop. For jobs-only stacks (no pods), the loop never executes, so
configmap files were never copied into the deployment directory.
The ConfigMaps were created as empty objects, leaving volume mounts
with no files.

Move the k8s configmap copying outside the pod loop so it runs
regardless of whether the stack has pods.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../deploy/deployment_create.py               | 25 +++++++++++--------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py
index 0546f370..21e5ca48 100644
--- a/stack_orchestrator/deploy/deployment_create.py
+++ b/stack_orchestrator/deploy/deployment_create.py
@@ -985,17 +985,7 @@ def _write_deployment_files(
             script_paths = get_pod_script_paths(parsed_stack, pod)
             _copy_files_to_directory(script_paths, destination_script_dir)
 
-        if parsed_spec.is_kubernetes_deployment():
-            for configmap in parsed_spec.get_configmaps():
-                source_config_dir = resolve_config_dir(stack_name, configmap)
-                if os.path.exists(source_config_dir):
-                    destination_config_dir = target_dir.joinpath(
-                        "configmaps", configmap
-                    )
-                    copytree(
-                        source_config_dir, destination_config_dir, dirs_exist_ok=True
-                    )
-        else:
+        if not parsed_spec.is_kubernetes_deployment():
             # TODO:
             # This is odd - looks up config dir that matches a volume name,
             # then copies as a mount dir?
@@ -1017,6 +1007,19 @@ def _write_deployment_files(
                             dirs_exist_ok=True,
                         )
 
+    # Copy configmap directories for k8s deployments (outside the pod loop
+    # so this works for jobs-only stacks too)
+    if parsed_spec.is_kubernetes_deployment():
+        for configmap in parsed_spec.get_configmaps():
+            source_config_dir = resolve_config_dir(stack_name, configmap)
+            if os.path.exists(source_config_dir):
+                destination_config_dir = target_dir.joinpath(
+                    "configmaps", configmap
+                )
+                copytree(
+                    source_config_dir, destination_config_dir, dirs_exist_ok=True
+                )
+
     # Copy the job files into the target dir
     jobs = get_job_list(parsed_stack)
     if jobs:
-- 
2.45.2


From b77037c73df78232a2805d059577954b0f99954e Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Fri, 6 Mar 2026 09:26:12 +0000
Subject: [PATCH 08/26] fix: remove shadowed os import in cluster_info

Inline `import os` at line 663 shadowed the top-level import,
causing flake8 F402.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 stack_orchestrator/deploy/k8s/cluster_info.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py
index be00142f..4ddf46d6 100644
--- a/stack_orchestrator/deploy/k8s/cluster_info.py
+++ b/stack_orchestrator/deploy/k8s/cluster_info.py
@@ -696,8 +696,6 @@ class ClusterInfo:
             )
 
             # Derive job name from file path: docker-compose-<name>.yml -> <name>
-            import os
-
             base = os.path.basename(job_file)
             # Strip docker-compose- prefix and .yml suffix
             job_name = base
-- 
2.45.2


From d7a742032e30add6fdc564e1373b6ad0845f8b14 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Fri, 6 Mar 2026 10:00:58 +0000
Subject: [PATCH 09/26] fix(webapp): use YAML round-trip instead of raw string
 append in _fixup_url_spec

The secrets: {} key added by init_operation for k8s deployments became
the last key in the spec file, breaking the raw string append that
assumed network: was always last. Replace with proper YAML load/modify/dump.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../deploy/webapp/deploy_webapp.py            | 26 +++++++++++--------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp.py b/stack_orchestrator/deploy/webapp/deploy_webapp.py
index 6170dbe3..8351fa0e 100644
--- a/stack_orchestrator/deploy/webapp/deploy_webapp.py
+++ b/stack_orchestrator/deploy/webapp/deploy_webapp.py
@@ -19,7 +19,7 @@ from pathlib import Path
 from urllib.parse import urlparse
 from tempfile import NamedTemporaryFile
 
-from stack_orchestrator.util import error_exit, global_options2
+from stack_orchestrator.util import error_exit, global_options2, get_yaml
 from stack_orchestrator.deploy.deployment_create import init_operation, create_operation
 from stack_orchestrator.deploy.deploy import create_deploy_context
 from stack_orchestrator.deploy.deploy_types import DeployCommandContext
@@ -41,19 +41,23 @@ def _fixup_container_tag(deployment_dir: str, image: str):
 def _fixup_url_spec(spec_file_name: str, url: str):
     # url is like: https://example.com/path
     parsed_url = urlparse(url)
-    http_proxy_spec = f"""
-  http-proxy:
-    - host-name: {parsed_url.hostname}
-      routes:
-        - path: '{parsed_url.path if parsed_url.path else "/"}'
-          proxy-to: webapp:80
-    """
     spec_file_path = Path(spec_file_name)
+    yaml = get_yaml()
     with open(spec_file_path) as rfile:
-        contents = rfile.read()
-        contents = contents + http_proxy_spec
+        contents = yaml.load(rfile)
+    contents.setdefault("network", {})["http-proxy"] = [
+        {
+            "host-name": parsed_url.hostname,
+            "routes": [
+                {
+                    "path": parsed_url.path if parsed_url.path else "/",
+                    "proxy-to": "webapp:80",
+                }
+            ],
+        }
+    ]
     with open(spec_file_path, "w") as wfile:
-        wfile.write(contents)
+        yaml.dump(contents, wfile)
 
 
 def create_deployment(
-- 
2.45.2


From b8702f0bfc40e3d467b794e0a2ea142a36223c32 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Mon, 9 Mar 2026 09:22:28 +0000
Subject: [PATCH 10/26] k8s: add app.kubernetes.io/stack label to pods and jobs

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 stack_orchestrator/deploy/k8s/cluster_info.py | 12 +++++++++---
 stack_orchestrator/deploy/k8s/deploy_k8s.py   |  2 ++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py
index 4ddf46d6..88fc5d3d 100644
--- a/stack_orchestrator/deploy/k8s/cluster_info.py
+++ b/stack_orchestrator/deploy/k8s/cluster_info.py
@@ -75,13 +75,14 @@ class ClusterInfo:
     parsed_job_yaml_map: Any
     image_set: Set[str] = set()
     app_name: str
+    stack_name: str
     environment_variables: DeployEnvVars
     spec: Spec
 
     def __init__(self) -> None:
         self.parsed_job_yaml_map = {}
 
-    def int(self, pod_files: List[str], compose_env_file, deployment_name, spec: Spec):
+    def int(self, pod_files: List[str], compose_env_file, deployment_name, spec: Spec, stack_name=""):
         self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files)
         # Find the set of images in the pods
         self.image_set = images_for_deployment(pod_files)
@@ -91,6 +92,7 @@ class ClusterInfo:
         }
         self.environment_variables = DeployEnvVars(env_vars)
         self.app_name = deployment_name
+        self.stack_name = stack_name
         self.spec = spec
         if opts.o.debug:
             print(f"Env vars: {self.environment_variables.map}")
@@ -589,6 +591,8 @@ class ClusterInfo:
 
         annotations = None
         labels = {"app": self.app_name}
+        if self.stack_name:
+            labels["app.kubernetes.io/stack"] = self.stack_name
         affinity = None
         tolerations = None
 
@@ -706,9 +710,10 @@ class ClusterInfo:
             elif job_name.endswith(".yaml"):
                 job_name = job_name[: -len(".yaml")]
 
+            pod_labels = {"app": self.app_name, **({"app.kubernetes.io/stack": self.stack_name} if self.stack_name else {})}
             template = client.V1PodTemplateSpec(
                 metadata=client.V1ObjectMeta(
-                    labels={"app": self.app_name}
+                    labels=pod_labels
                 ),
                 spec=client.V1PodSpec(
                     containers=containers,
@@ -721,12 +726,13 @@ class ClusterInfo:
                 template=template,
                 backoff_limit=0,
             )
+            job_labels = {"app": self.app_name, **({"app.kubernetes.io/stack": self.stack_name} if self.stack_name else {})}
             job = client.V1Job(
                 api_version="batch/v1",
                 kind="Job",
                 metadata=client.V1ObjectMeta(
                     name=f"{self.app_name}-job-{job_name}",
-                    labels={"app": self.app_name},
+                    labels=job_labels,
                 ),
                 spec=job_spec,
             )
diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py
index 55513d6e..1efdf177 100644
--- a/stack_orchestrator/deploy/k8s/deploy_k8s.py
+++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py
@@ -126,11 +126,13 @@ class K8sDeployer(Deployer):
         # Use deployment-specific namespace for resource isolation and easy cleanup
         self.k8s_namespace = f"laconic-{compose_project_name}"
         self.cluster_info = ClusterInfo()
+        stack_name = deployment_context.stack.name if deployment_context else ""
         self.cluster_info.int(
             compose_files,
             compose_env_file,
             compose_project_name,
             deployment_context.spec,
+            stack_name=stack_name,
         )
         # Initialize job compose files if provided
         if job_compose_files:
-- 
2.45.2


From 7c8a4d91e72c831073659932af1072db82825eb7 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Mon, 9 Mar 2026 09:49:24 +0000
Subject: [PATCH 11/26] k8s: add start() hook for post-deployment k8s resource
 creation

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../deploy/deployment_create.py               | 19 +++++++++++++++++++
 stack_orchestrator/deploy/k8s/deploy_k8s.py   |  5 +++++
 2 files changed, 24 insertions(+)

diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py
index 21e5ca48..792d8e3d 100644
--- a/stack_orchestrator/deploy/deployment_create.py
+++ b/stack_orchestrator/deploy/deployment_create.py
@@ -265,6 +265,25 @@ def call_stack_deploy_create(deployment_context, extra_args):
                 imported_stack.create(deployment_context, extra_args)
 
 
+def call_stack_deploy_start(deployment_context):
+    """Call start() hooks after k8s deployments and jobs are created.
+
+    The start() hook receives the DeploymentContext, allowing stacks to
+    create additional k8s resources (Services, etc.) in the deployment namespace.
+    The namespace can be derived as f"laconic-{deployment_context.id}".
+    """
+    python_file_paths = _commands_plugin_paths(deployment_context.stack.name)
+    for python_file_path in python_file_paths:
+        if python_file_path.exists():
+            spec = util.spec_from_file_location("commands", python_file_path)
+            if spec is None or spec.loader is None:
+                continue
+            imported_stack = util.module_from_spec(spec)
+            spec.loader.exec_module(imported_stack)
+            if _has_method(imported_stack, "start"):
+                imported_stack.start(deployment_context)
+
+
 # Inspect the pod yaml to find config files referenced in subdirectories
 # other than the one associated with the pod
 def _find_extra_config_dirs(parsed_pod_file, pod):
diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py
index 1efdf177..3003129c 100644
--- a/stack_orchestrator/deploy/k8s/deploy_k8s.py
+++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py
@@ -461,6 +461,11 @@ class K8sDeployer(Deployer):
                     print("NodePort created:")
                     print(f"{nodeport_resp}")
 
+        # Call start() hooks — stacks can create additional k8s resources
+        if self.deployment_context:
+            from stack_orchestrator.deploy.deployment_create import call_stack_deploy_start
+            call_stack_deploy_start(self.deployment_context)
+
     def down(self, timeout, volumes, skip_cluster_management):
         self.skip_cluster_management = skip_cluster_management
         self.connect_api()
-- 
2.45.2


From ef07b2c86eaf7b9f2fbd13b5d87b91a7a32f2d17 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Mon, 9 Mar 2026 12:08:53 +0000
Subject: [PATCH 12/26] k8s: extract basename from stack path for labels

Stack.name contains the full absolute path from the spec file's
"stack:" key (e.g. /home/.../stacks/hyperlane-minio). K8s labels
must be <= 63 bytes and alphanumeric. Extract just the directory
basename (e.g. "hyperlane-minio") before using it as a label value.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 stack_orchestrator/deploy/k8s/deploy_k8s.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py
index 3003129c..c13f9a55 100644
--- a/stack_orchestrator/deploy/k8s/deploy_k8s.py
+++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py
@@ -126,7 +126,10 @@ class K8sDeployer(Deployer):
         # Use deployment-specific namespace for resource isolation and easy cleanup
         self.k8s_namespace = f"laconic-{compose_project_name}"
         self.cluster_info = ClusterInfo()
-        stack_name = deployment_context.stack.name if deployment_context else ""
+        # stack.name may be an absolute path (from spec "stack:" key after
+        # path resolution). Extract just the directory basename for labels.
+        raw_name = deployment_context.stack.name if deployment_context else ""
+        stack_name = Path(raw_name).name if raw_name else ""
         self.cluster_info.int(
             compose_files,
             compose_env_file,
-- 
2.45.2


From 517e102830021b3f1a2c69406e013920547b6fdd Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Mon, 9 Mar 2026 12:56:27 +0000
Subject: [PATCH 13/26] fix(k8s): use deployment namespace for pod/container
 lookups

pods_in_deployment() and containers_in_pod() hardcoded
namespace="default", but pods are created in the deployment-specific
namespace (laconic-{cluster-id}). This caused logs() to return
"Pods not running" even when pods were healthy.

Add namespace parameter to both functions and pass
self.k8s_namespace from the logs() caller.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 stack_orchestrator/deploy/k8s/deploy_k8s.py | 4 ++--
 stack_orchestrator/deploy/k8s/helpers.py    | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py
index c13f9a55..9ad61838 100644
--- a/stack_orchestrator/deploy/k8s/deploy_k8s.py
+++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py
@@ -617,14 +617,14 @@ class K8sDeployer(Deployer):
 
     def logs(self, services, tail, follow, stream):
         self.connect_api()
-        pods = pods_in_deployment(self.core_api, self.cluster_info.app_name)
+        pods = pods_in_deployment(self.core_api, self.cluster_info.app_name, namespace=self.k8s_namespace)
         if len(pods) > 1:
             print("Warning: more than one pod in the deployment")
         if len(pods) == 0:
             log_data = "******* Pods not running ********\n"
         else:
             k8s_pod_name = pods[0]
-            containers = containers_in_pod(self.core_api, k8s_pod_name)
+            containers = containers_in_pod(self.core_api, k8s_pod_name, namespace=self.k8s_namespace)
             # If pod not started, logs request below will throw an exception
             try:
                 log_data = ""
diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py
index 8b367f86..1eedfd5f 100644
--- a/stack_orchestrator/deploy/k8s/helpers.py
+++ b/stack_orchestrator/deploy/k8s/helpers.py
@@ -393,10 +393,10 @@ def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]):
             raise DeployerException(f"kind load docker-image failed: {result}")
 
 
-def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
+def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str, namespace: str = "default"):
     pods = []
     pod_response = core_api.list_namespaced_pod(
-        namespace="default", label_selector=f"app={deployment_name}"
+        namespace=namespace, label_selector=f"app={deployment_name}"
     )
     if opts.o.debug:
         print(f"pod_response: {pod_response}")
@@ -406,10 +406,10 @@ def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
     return pods
 
 
-def containers_in_pod(core_api: client.CoreV1Api, pod_name: str) -> List[str]:
+def containers_in_pod(core_api: client.CoreV1Api, pod_name: str, namespace: str = "default") -> List[str]:
     containers: List[str] = []
     pod_response = cast(
-        client.V1Pod, core_api.read_namespaced_pod(pod_name, namespace="default")
+        client.V1Pod, core_api.read_namespaced_pod(pod_name, namespace=namespace)
     )
     if opts.o.debug:
         print(f"pod_response: {pod_response}")
-- 
2.45.2


From 183a1888747b97d963af1541155af4ca1160fac5 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Mon, 9 Mar 2026 13:18:36 +0000
Subject: [PATCH 14/26] ci: upgrade Kind to v0.25.0 and pin kubectl to v1.31.2

Kind v0.20.0 defaults to k8s v1.27.3 which fails on newer CI runners
(kubelet cgroups issue). Upgrade to Kind v0.25.0 (k8s v1.31.2) and
pin kubectl to match.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 tests/scripts/install-kind.sh    | 2 +-
 tests/scripts/install-kubectl.sh | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/scripts/install-kind.sh b/tests/scripts/install-kind.sh
index 254c3288..3b2debe4 100755
--- a/tests/scripts/install-kind.sh
+++ b/tests/scripts/install-kind.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 # TODO: handle ARM
-curl --silent -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
+curl --silent -Lo ./kind https://kind.sigs.k8s.io/dl/v0.25.0/kind-linux-amd64
 chmod +x ./kind
 mv ./kind /usr/local/bin
diff --git a/tests/scripts/install-kubectl.sh b/tests/scripts/install-kubectl.sh
index 7a5062fe..79890a6b 100755
--- a/tests/scripts/install-kubectl.sh
+++ b/tests/scripts/install-kubectl.sh
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 # TODO: handle ARM
-curl --silent -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+# Pin kubectl to match Kind's default k8s version (v1.31.x)
+curl --silent -LO "https://dl.k8s.io/release/v1.31.2/bin/linux/amd64/kubectl"
 chmod +x ./kubectl
 mv ./kubectl /usr/local/bin
-- 
2.45.2


From 241cd75671963205457290d6f12afff952da0e74 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 10 Mar 2026 04:53:52 +0000
Subject: [PATCH 15/26] fix(test): use deployment namespace in k8s control test

The deployment control test queries pods with raw kubectl but didn't
specify the namespace. Since pods now live in laconic-{deployment_id}
instead of default, the query returned empty results.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 tests/k8s-deployment-control/run-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/k8s-deployment-control/run-test.sh b/tests/k8s-deployment-control/run-test.sh
index c6e43694..cecad44d 100755
--- a/tests/k8s-deployment-control/run-test.sh
+++ b/tests/k8s-deployment-control/run-test.sh
@@ -206,7 +206,7 @@ fi
 # The deployment's pod should be scheduled onto node: worker3
 # Check that's what happened
 # Get get the node onto which the stack pod has been deployed
-deployment_node=$(kubectl get pods -l app=${deployment_id} -o=jsonpath='{.items..spec.nodeName}')
+deployment_node=$(kubectl get pods -n laconic-${deployment_id} -l app=${deployment_id} -o=jsonpath='{.items..spec.nodeName}')
 expected_node=${deployment_id}-worker3
 echo "Stack pod deployed to node: ${deployment_node}"
 if [[ ${deployment_node} == ${expected_node} ]]; then
-- 
2.45.2


From 1375f209d3be0ce923e909c194ef66905778789d Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 10 Mar 2026 05:06:31 +0000
Subject: [PATCH 16/26] test(k8s): add tests for jobs, secrets, labels, and
 namespace isolation

Add a job compose file for the test stack and extend the k8s deploy
test to verify new features:
- Namespace isolation: pod exists in laconic-{id}, not default
- Stack labels: app.kubernetes.io/stack label set on pods
- Job completion: test-job runs to completion (status.succeeded=1)
- Secrets: spec secrets: key results in envFrom secretRef on pod

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../compose-jobs/docker-compose-test-job.yml  |  5 ++
 stack_orchestrator/data/stacks/test/stack.yml |  2 +
 tests/k8s-deploy/run-deploy-test.sh           | 59 +++++++++++++++++++
 3 files changed, 66 insertions(+)
 create mode 100644 stack_orchestrator/data/compose-jobs/docker-compose-test-job.yml

diff --git a/stack_orchestrator/data/compose-jobs/docker-compose-test-job.yml b/stack_orchestrator/data/compose-jobs/docker-compose-test-job.yml
new file mode 100644
index 00000000..10ccf4b4
--- /dev/null
+++ b/stack_orchestrator/data/compose-jobs/docker-compose-test-job.yml
@@ -0,0 +1,5 @@
+services:
+  test-job:
+    image: cerc/test-container:local
+    entrypoint: /bin/sh
+    command: ["-c", "echo 'Job completed successfully'"]
diff --git a/stack_orchestrator/data/stacks/test/stack.yml b/stack_orchestrator/data/stacks/test/stack.yml
index 93d3ecd3..224590ff 100644
--- a/stack_orchestrator/data/stacks/test/stack.yml
+++ b/stack_orchestrator/data/stacks/test/stack.yml
@@ -7,3 +7,5 @@ containers:
   - cerc/test-container
 pods:
   - test
+jobs:
+  - test-job
diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh
index e482a5b7..5cee371a 100755
--- a/tests/k8s-deploy/run-deploy-test.sh
+++ b/tests/k8s-deploy/run-deploy-test.sh
@@ -105,6 +105,17 @@ fi
 # Add a config file to be picked up by the ConfigMap before starting.
 echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/configmaps/test-config/test_config
 
+# Add secrets to the deployment spec (references a pre-existing k8s Secret by name)
+deployment_spec_file=${test_deployment_dir}/spec.yml
+cat << EOF >> ${deployment_spec_file}
+secrets:
+  test-secret:
+    - TEST_SECRET_KEY
+EOF
+
+# Get the deployment ID for kubectl queries
+deployment_id=$(cat ${test_deployment_dir}/deployment.yml | cut -d ' ' -f 2)
+
 echo "deploy create output file test: passed"
 # Try to start the deployment
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
@@ -166,6 +177,54 @@ else
     delete_cluster_exit
 fi
 
+# --- New feature tests: namespace, labels, jobs, secrets ---
+
+# Check that the pod is in the deployment-specific namespace (not default)
+ns_pod_count=$(kubectl get pods -n laconic-${deployment_id} -l app=${deployment_id} --no-headers 2>/dev/null | wc -l)
+if [ "$ns_pod_count" -gt 0 ]; then
+    echo "namespace isolation test: passed"
+else
+    echo "namespace isolation test: FAILED"
+    echo "Expected pod in namespace laconic-${deployment_id}"
+    delete_cluster_exit
+fi
+
+# Check that the stack label is set on the pod
+stack_label_count=$(kubectl get pods -n laconic-${deployment_id} -l app.kubernetes.io/stack=test --no-headers 2>/dev/null | wc -l)
+if [ "$stack_label_count" -gt 0 ]; then
+    echo "stack label test: passed"
+else
+    echo "stack label test: FAILED"
+    delete_cluster_exit
+fi
+
+# Check that the job completed successfully
+for i in {1..30}; do
+    job_status=$(kubectl get job ${deployment_id}-job-test-job -n laconic-${deployment_id} -o jsonpath='{.status.succeeded}' 2>/dev/null)
+    if [ "$job_status" == "1" ]; then
+        break
+    fi
+    sleep 2
+done
+if [ "$job_status" == "1" ]; then
+    echo "job completion test: passed"
+else
+    echo "job completion test: FAILED"
+    echo "Job status.succeeded: ${job_status}"
+    delete_cluster_exit
+fi
+
+# Check that the secrets spec results in an envFrom secretRef on the pod
+secret_ref=$(kubectl get pod -n laconic-${deployment_id} -l app=${deployment_id} \
+    -o jsonpath='{.items[0].spec.containers[0].envFrom[?(@.secretRef.name=="test-secret")].secretRef.name}' 2>/dev/null)
+if [ "$secret_ref" == "test-secret" ]; then
+    echo "secrets envFrom test: passed"
+else
+    echo "secrets envFrom test: FAILED"
+    echo "Expected secretRef 'test-secret', got: ${secret_ref}"
+    delete_cluster_exit
+fi
+
 # Stop then start again and check the volume was preserved
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop
 # Sleep a bit just in case
-- 
2.45.2


From 35f179b755d8bad1fd60b8f859f06955d713748c Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 10 Mar 2026 05:13:26 +0000
Subject: [PATCH 17/26] fix(test): reuse kind cluster on stop/start cycle in
 deploy test

Use --skip-cluster-management to avoid destroying and recreating the
kind cluster during the stop/start volume retention test. The second
kind create fails on some CI runners due to cgroups detection issues.

Use --delete-volumes to clear PVs so fresh PVCs can bind on restart.
Bind-mount data survives on the host filesystem; provisioner volumes
are recreated fresh.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 tests/k8s-deploy/run-deploy-test.sh | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh
index 5cee371a..50d0907f 100755
--- a/tests/k8s-deploy/run-deploy-test.sh
+++ b/tests/k8s-deploy/run-deploy-test.sh
@@ -225,12 +225,14 @@ else
     delete_cluster_exit
 fi
 
-# Stop then start again and check the volume was preserved
-$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
-# Sleep a bit just in case
-# sleep for longer to check if that's why the subsequent create cluster fails
-sleep 20
-$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+# Stop then start again and check the volume was preserved.
+# Use --skip-cluster-management to reuse the existing kind cluster instead of
+# destroying and recreating it (which fails on some CI runners due to cgroups issues).
+# Use --delete-volumes to clear PVs so fresh PVCs can bind on restart.
+# Bind-mount data survives on the host filesystem; provisioner volumes are recreated fresh.
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes --skip-cluster-management
+sleep 5
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start --skip-cluster-management
 wait_for_pods_started
 wait_for_log_output
 sleep 1
-- 
2.45.2


From d64046df55843774e8d1670f3300409a41a4b29f Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 10 Mar 2026 05:24:00 +0000
Subject: [PATCH 18/26] Revert "fix(test): reuse kind cluster on stop/start
 cycle in deploy test"

This reverts commit 35f179b755d8bad1fd60b8f859f06955d713748c.
---
 tests/k8s-deploy/run-deploy-test.sh | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh
index 50d0907f..5cee371a 100755
--- a/tests/k8s-deploy/run-deploy-test.sh
+++ b/tests/k8s-deploy/run-deploy-test.sh
@@ -225,14 +225,12 @@ else
     delete_cluster_exit
 fi
 
-# Stop then start again and check the volume was preserved.
-# Use --skip-cluster-management to reuse the existing kind cluster instead of
-# destroying and recreating it (which fails on some CI runners due to cgroups issues).
-# Use --delete-volumes to clear PVs so fresh PVCs can bind on restart.
-# Bind-mount data survives on the host filesystem; provisioner volumes are recreated fresh.
-$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes --skip-cluster-management
-sleep 5
-$TEST_TARGET_SO deployment --dir $test_deployment_dir start --skip-cluster-management
+# Stop then start again and check the volume was preserved
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
+# Sleep a bit just in case
+# sleep for longer to check if that's why the subsequent create cluster fails
+sleep 20
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
 wait_for_pods_started
 wait_for_log_output
 sleep 1
-- 
2.45.2


From 108f13a09b66d4c9ee5c4cfa5c2b17985fc28227 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 10 Mar 2026 05:26:48 +0000
Subject: [PATCH 19/26] fix(test): wait for kind cluster cleanup before
 recreating

Replace the fixed `sleep 20` with a polling loop that waits for
`kind get clusters` to report no clusters. The previous approach
was flaky on CI runners where Docker takes longer to tear down
cgroup hierarchies after `kind delete cluster`.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 tests/k8s-deploy/run-deploy-test.sh | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh
index 5cee371a..fdc5a1db 100755
--- a/tests/k8s-deploy/run-deploy-test.sh
+++ b/tests/k8s-deploy/run-deploy-test.sh
@@ -46,6 +46,18 @@ wait_for_log_output () {
 }
 
 
+wait_for_cluster_destroyed () {
+    for i in {1..60}
+    do
+        if ! kind get clusters 2>/dev/null | grep -q .; then
+            return
+        fi
+        sleep 2
+    done
+    echo "waiting for kind cluster cleanup: FAILED"
+    exit 1
+}
+
 delete_cluster_exit () {
     $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
     exit 1
@@ -227,9 +239,10 @@ fi
 
 # Stop then start again and check the volume was preserved
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop
-# Sleep a bit just in case
-# sleep for longer to check if that's why the subsequent create cluster fails
-sleep 20
+# Wait for the kind cluster to be fully destroyed before recreating it.
+# Without this, the second 'kind create cluster' can fail with cgroup
+# detection errors because Docker hasn't finished cleaning up.
+wait_for_cluster_destroyed
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
 wait_for_pods_started
 wait_for_log_output
-- 
2.45.2


From 464215c72a34775a40169e81243ce667b23501c2 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 10 Mar 2026 05:34:37 +0000
Subject: [PATCH 20/26] fix(test): replace empty secrets key instead of
 appending duplicate

deploy init already writes 'secrets: {}' into the spec file. The test
was appending a second secrets block via heredoc, which ruamel.yaml
rejects as a duplicate key. Use sed to replace the empty value instead.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 tests/k8s-deploy/run-deploy-test.sh | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh
index fdc5a1db..c6836ee1 100755
--- a/tests/k8s-deploy/run-deploy-test.sh
+++ b/tests/k8s-deploy/run-deploy-test.sh
@@ -117,13 +117,11 @@ fi
 # Add a config file to be picked up by the ConfigMap before starting.
 echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/configmaps/test-config/test_config
 
-# Add secrets to the deployment spec (references a pre-existing k8s Secret by name)
+# Add secrets to the deployment spec (references a pre-existing k8s Secret by name).
+# deploy init already writes an empty 'secrets: {}' key, so we replace it
+# rather than appending (ruamel.yaml rejects duplicate keys).
 deployment_spec_file=${test_deployment_dir}/spec.yml
-cat << EOF >> ${deployment_spec_file}
-secrets:
-  test-secret:
-    - TEST_SECRET_KEY
-EOF
+sed -i 's/^secrets: {}$/secrets:\n  test-secret:\n    - TEST_SECRET_KEY/' ${deployment_spec_file}
 
 # Get the deployment ID for kubectl queries
 deployment_id=$(cat ${test_deployment_dir}/deployment.yml | cut -d ' ' -f 2)
-- 
2.45.2


From a1b5220e40ad0f569f0ce839ee342014c92f4dc6 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 10 Mar 2026 05:59:35 +0000
Subject: [PATCH 21/26] fix(test): prevent set -e from killing kubectl queries
 in test checks

kubectl commands that query jobs or pod specs exit non-zero when the
resource doesn't exist yet. Under set -e, a bare command substitution
like var=$(kubectl ...) aborts the script silently. Add || true so
the polling loop and assertion logic can handle failures gracefully.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 tests/k8s-deploy/run-deploy-test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh
index c6836ee1..39d50eba 100755
--- a/tests/k8s-deploy/run-deploy-test.sh
+++ b/tests/k8s-deploy/run-deploy-test.sh
@@ -210,7 +210,7 @@ fi
 
 # Check that the job completed successfully
 for i in {1..30}; do
-    job_status=$(kubectl get job ${deployment_id}-job-test-job -n laconic-${deployment_id} -o jsonpath='{.status.succeeded}' 2>/dev/null)
+    job_status=$(kubectl get job ${deployment_id}-job-test-job -n laconic-${deployment_id} -o jsonpath='{.status.succeeded}' 2>/dev/null || true)
     if [ "$job_status" == "1" ]; then
         break
     fi
@@ -226,7 +226,7 @@ fi
 
 # Check that the secrets spec results in an envFrom secretRef on the pod
 secret_ref=$(kubectl get pod -n laconic-${deployment_id} -l app=${deployment_id} \
-    -o jsonpath='{.items[0].spec.containers[0].envFrom[?(@.secretRef.name=="test-secret")].secretRef.name}' 2>/dev/null)
+    -o jsonpath='{.items[0].spec.containers[0].envFrom[?(@.secretRef.name=="test-secret")].secretRef.name}' 2>/dev/null || true)
 if [ "$secret_ref" == "test-secret" ]; then
     echo "secrets envFrom test: passed"
 else
-- 
2.45.2


From 68ef9de0161dbe5f2008eea3dbb91865928b3869 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 10 Mar 2026 06:15:15 +0000
Subject: [PATCH 22/26] fix(k8s): resolve internal job compose files from
 data/compose-jobs

resolve_job_compose_file() used Path(stack).parent.parent for the
internal fallback, which resolved to data/stacks/compose-jobs/ instead
of data/compose-jobs/. This meant deploy create couldn't find job
compose files for internal stacks, so they were never copied to the
deployment directory and never created as k8s Jobs.

Use the same data directory resolution pattern as resolve_compose_file.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 stack_orchestrator/util.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/stack_orchestrator/util.py b/stack_orchestrator/util.py
index 7e1e442c..3e0fb3f1 100644
--- a/stack_orchestrator/util.py
+++ b/stack_orchestrator/util.py
@@ -155,9 +155,8 @@ def resolve_job_compose_file(stack, job_name: str):
         if proposed_file.exists():
             return proposed_file
         # If we don't find it fall through to the internal case
-    # TODO: Add internal compose-jobs directory support if needed
-    # For now, jobs are expected to be in external stacks only
-    compose_jobs_base = Path(stack).parent.parent.joinpath("compose-jobs")
+    data_dir = Path(__file__).absolute().parent.joinpath("data")
+    compose_jobs_base = data_dir.joinpath("compose-jobs")
     return compose_jobs_base.joinpath(f"docker-compose-{job_name}.yml")
 
 
-- 
2.45.2


From 91f4e5fe38f784697baff517cc80f081d10afad5 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 10 Mar 2026 06:26:03 +0000
Subject: [PATCH 23/26] fix(k8s): use distinct app label for job pods

Job pod templates used the same app={deployment_id} label as
deployment pods, causing pods_in_deployment() to return both.
This made the logs command warn about multiple pods and pick
the wrong one.

Use app={deployment_id}-job for job pod templates so they are
not matched by pods_in_deployment(). The Job metadata itself
retains the original app label for stack-level queries.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 stack_orchestrator/deploy/k8s/cluster_info.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py
index 88fc5d3d..e773a346 100644
--- a/stack_orchestrator/deploy/k8s/cluster_info.py
+++ b/stack_orchestrator/deploy/k8s/cluster_info.py
@@ -710,7 +710,9 @@ class ClusterInfo:
             elif job_name.endswith(".yaml"):
                 job_name = job_name[: -len(".yaml")]
 
-            pod_labels = {"app": self.app_name, **({"app.kubernetes.io/stack": self.stack_name} if self.stack_name else {})}
+            # Use a distinct app label for job pods so they don't get
+            # picked up by pods_in_deployment() which queries app={app_name}.
+            pod_labels = {"app": f"{self.app_name}-job", **({"app.kubernetes.io/stack": self.stack_name} if self.stack_name else {})}
             template = client.V1PodTemplateSpec(
                 metadata=client.V1ObjectMeta(
                     labels=pod_labels
-- 
2.45.2


From a1c6c35834a38b44fb6b8e5aac68bbed362f866e Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 10 Mar 2026 06:31:25 +0000
Subject: [PATCH 24/26] style: wrap long line in cluster_info.py to fix flake8
 E501

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 stack_orchestrator/deploy/k8s/cluster_info.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py
index e773a346..8c530fc9 100644
--- a/stack_orchestrator/deploy/k8s/cluster_info.py
+++ b/stack_orchestrator/deploy/k8s/cluster_info.py
@@ -712,7 +712,10 @@ class ClusterInfo:
 
             # Use a distinct app label for job pods so they don't get
             # picked up by pods_in_deployment() which queries app={app_name}.
-            pod_labels = {"app": f"{self.app_name}-job", **({"app.kubernetes.io/stack": self.stack_name} if self.stack_name else {})}
+            pod_labels = {
+                "app": f"{self.app_name}-job",
+                **({"app.kubernetes.io/stack": self.stack_name} if self.stack_name else {}),
+            }
             template = client.V1PodTemplateSpec(
                 metadata=client.V1ObjectMeta(
                     labels=pod_labels
-- 
2.45.2


From b85c12e4da87adef9bb41d4fb5c532cc5d37ba6f Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 10 Mar 2026 06:49:42 +0000
Subject: [PATCH 25/26] fix(test): use --skip-cluster-management for stop/start
 volume test

Recreating a kind cluster in the same CI run fails due to stale
etcd/certs and cgroup detection issues. Use --skip-cluster-management
to reuse the existing cluster, and --delete-volumes to clear PVs so
fresh PVCs can bind on restart.

The volume retention semantics are preserved: bind-mount host path
data survives (filesystem is old), provisioner volumes are fresh
(PVs were deleted).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 tests/k8s-deploy/run-deploy-test.sh | 33 +++++++++++------------------
 1 file changed, 12 insertions(+), 21 deletions(-)

diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh
index 39d50eba..942110d4 100755
--- a/tests/k8s-deploy/run-deploy-test.sh
+++ b/tests/k8s-deploy/run-deploy-test.sh
@@ -46,18 +46,6 @@ wait_for_log_output () {
 }
 
 
-wait_for_cluster_destroyed () {
-    for i in {1..60}
-    do
-        if ! kind get clusters 2>/dev/null | grep -q .; then
-            return
-        fi
-        sleep 2
-    done
-    echo "waiting for kind cluster cleanup: FAILED"
-    exit 1
-}
-
 delete_cluster_exit () {
     $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
     exit 1
@@ -235,13 +223,15 @@ else
     delete_cluster_exit
 fi
 
-# Stop then start again and check the volume was preserved
-$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
-# Wait for the kind cluster to be fully destroyed before recreating it.
-# Without this, the second 'kind create cluster' can fail with cgroup
-# detection errors because Docker hasn't finished cleaning up.
-wait_for_cluster_destroyed
-$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+# Stop then start again and check the volume was preserved.
+# Use --skip-cluster-management to reuse the existing kind cluster instead of
+# destroying and recreating it (which fails on CI runners due to stale etcd/certs
+# and cgroup detection issues).
+# Use --delete-volumes to clear PVs so fresh PVCs can bind on restart.
+# Bind-mount data survives on the host filesystem; provisioner volumes are recreated fresh.
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes --skip-cluster-management
+sleep 5
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start --skip-cluster-management
 wait_for_pods_started
 wait_for_log_output
 sleep 1
@@ -254,8 +244,9 @@ else
     delete_cluster_exit
 fi
 
-# These volumes will be completely destroyed by the kind delete/create, because they lived inside
-# the kind container.  So, unlike the bind-mount case, they will appear fresh after the restart.
+# Provisioner volumes are destroyed when PVs are deleted (--delete-volumes on stop).
+# Unlike bind-mount volumes whose data persists on the host, provisioner storage
+# is gone, so the volume appears fresh after restart.
 log_output_11=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
 if [[ "$log_output_11" == *"/data2 filesystem is fresh"* ]]; then
     echo "Fresh provisioner volumes test: passed"
-- 
2.45.2


From aac317503ea2eba68439bf1d49e413d26f2aa96c Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Tue, 10 Mar 2026 07:01:27 +0000
Subject: [PATCH 26/26] fix(test): wait for namespace termination before
 restart

Replace fixed sleep with a polling loop that waits for the deployment
namespace to be fully deleted. Without this, the start command fails
with 403 Forbidden because k8s rejects resource creation in a
namespace that is still terminating.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 tests/k8s-deploy/run-deploy-test.sh | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh
index 942110d4..cfc03138 100755
--- a/tests/k8s-deploy/run-deploy-test.sh
+++ b/tests/k8s-deploy/run-deploy-test.sh
@@ -230,7 +230,15 @@ fi
 # Use --delete-volumes to clear PVs so fresh PVCs can bind on restart.
 # Bind-mount data survives on the host filesystem; provisioner volumes are recreated fresh.
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes --skip-cluster-management
-sleep 5
+# Wait for the namespace to be fully terminated before restarting.
+# Without this, 'start' fails with 403 Forbidden because the namespace
+# is still in Terminating state.
+for i in {1..60}; do
+    if ! kubectl get namespace laconic-${deployment_id} 2>/dev/null | grep -q .; then
+        break
+    fi
+    sleep 2
+done
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start --skip-cluster-management
 wait_for_pods_started
 wait_for_log_output
-- 
2.45.2