Merge branch '_so_push' into _so_main_push

Fixes ReDoS vulnerability in pydantic email validation regex.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.gitea/workflows/triggers/test-database

@@ -1,2 +1,2 @@
-Change this file to trigger running the test-database CI job
+Change this file to trigger running the test-database CI job
 Trigger test run

.gitignore

@@ -8,3 +8,4 @@ __pycache__
 package
 stack_orchestrator/data/build_tag.txt
 /build
+.worktrees

.pebbles/config.json

@@ -0,0 +1,3 @@
+{
+  "prefix": "so"
+}

.pebbles/events.jsonl

@@ -0,0 +1,15 @@
+{"type":"create","timestamp":"2026-03-08T06:56:07.080584539Z","issue_id":"so-076","payload":{"description":"Currently laconic-so maps one stack to one deployment to one pod. All containers\nin a stack's compose files become containers in a single k8s pod. This means:\n\n- Can't upgrade doublezero without restarting agave-validator\n- Can't restart monitoring without disrupting the validator\n- Can't independently scale or lifecycle-manage components\n\nThe fix is stack composition. A meta-stack (e.g. biscayne-stack) composes\nsub-stacks (agave, doublezero, agave-monitoring), each becoming its own\nk8s Deployment with independent lifecycle.","priority":"2","title":"Stack composition: deploy multiple stacks into one kind cluster","type":"epic"}}
+{"type":"create","timestamp":"2026-03-08T06:56:07.551986919Z","issue_id":"so-ab0","payload":{"description":"Add laconic-so deployment prepare that creates cluster infrastructure without pods. Already implemented, needs review.","priority":"2","title":"deployment prepare command","type":"task"}}
+{"type":"create","timestamp":"2026-03-08T06:56:07.884418759Z","issue_id":"so-04f","payload":{"description":"deployment stop on ANY deployment deletes the shared kind cluster. Should only delete its own namespace.","priority":"2","title":"deployment stop should not destroy shared cluster","type":"bug"}}
+{"type":"create","timestamp":"2026-03-08T06:56:08.253520249Z","issue_id":"so-370","payload":{"description":"Allow stack.yml to reference sub-stacks. Each sub-stack becomes its own k8s Deployment sharing namespace and PVs.","priority":"2","title":"Add stacks: field to stack.yml for composition","type":"task"}}
+{"type":"create","timestamp":"2026-03-08T06:56:08.646764337Z","issue_id":"so-f7c","payload":{"description":"Create three independent stacks from the monolithic agave-stack. Each gets its own compose file and independent lifecycle.","priority":"2","title":"Split agave-stack into agave + doublezero + monitoring","type":"task"}}
+{"type":"rename","timestamp":"2026-03-08T06:56:14.499990161Z","issue_id":"so-ab0","payload":{"new_id":"so-076.1"}}
+{"type":"dep_add","timestamp":"2026-03-08T06:56:14.499992031Z","issue_id":"so-076.1","payload":{"dep_type":"parent-child","depends_on":"so-076"}}
+{"type":"rename","timestamp":"2026-03-08T06:56:14.786407752Z","issue_id":"so-04f","payload":{"new_id":"so-076.2"}}
+{"type":"dep_add","timestamp":"2026-03-08T06:56:14.786409842Z","issue_id":"so-076.2","payload":{"dep_type":"parent-child","depends_on":"so-076"}}
+{"type":"rename","timestamp":"2026-03-08T06:56:15.058959714Z","issue_id":"so-370","payload":{"new_id":"so-076.3"}}
+{"type":"dep_add","timestamp":"2026-03-08T06:56:15.058961364Z","issue_id":"so-076.3","payload":{"dep_type":"parent-child","depends_on":"so-076"}}
+{"type":"rename","timestamp":"2026-03-08T06:56:15.410080785Z","issue_id":"so-f7c","payload":{"new_id":"so-076.4"}}
+{"type":"dep_add","timestamp":"2026-03-08T06:56:15.410082305Z","issue_id":"so-076.4","payload":{"dep_type":"parent-child","depends_on":"so-076"}}
+{"type":"dep_add","timestamp":"2026-03-08T06:56:16.313585082Z","issue_id":"so-076.3","payload":{"dep_type":"blocks","depends_on":"so-076.2"}}
+{"type":"dep_add","timestamp":"2026-03-08T06:56:16.567629422Z","issue_id":"so-076.4","payload":{"dep_type":"blocks","depends_on":"so-076.3"}}

docs/cli.md

@@ -68,7 +68,7 @@ $ laconic-so build-npms --include <package-name> --force-rebuild
 
 ## deploy
 
-The `deploy` command group manages persistent deployments. The general workflow is `deploy init` to generate a spec file, then `deploy create` to create a deployment directory from the spec, then runtime commands like `deployment start` and `deployment stop`.
+The `deploy` command group manages persistent deployments. The general workflow is `deploy init` to generate a spec file, then `deploy create` to create a deployment directory from the spec, then runtime commands like `deploy up` and `deploy down`.
 
 ### deploy init
 
@@ -101,91 +101,35 @@ Options:
 - `--spec-file` (required): spec file to use
 - `--deployment-dir`: target directory for deployment files
 - `--update`: update an existing deployment directory, preserving data volumes and env file. Changed files are backed up with a `.bak` suffix. The deployment's `config.env` and `deployment.yml` are also preserved.
-- `--helm-chart`: generate Helm chart instead of deploying (k8s only)
 - `--network-dir`: network configuration supplied in this directory
 - `--initial-peers`: initial set of persistent peers
 
-## deployment
+### deploy up
 
-Runtime commands for managing a created deployment. Use `--dir` to specify the deployment directory.
-
-### deployment start
-
-Start a deployment (`up` is a legacy alias):
+Start a deployment:
 ```
-$ laconic-so deployment --dir <deployment-dir> start
+$ laconic-so deployment --dir <deployment-dir> up
 ```
 
-Options:
-- `--stay-attached` / `--detatch-terminal`: attach to container stdout (default: detach)
-- `--skip-cluster-management` / `--perform-cluster-management`: skip kind cluster creation/teardown (default: perform management). Only affects k8s-kind deployments. Use this when multiple stacks share a single cluster.
+### deploy down
 
-### deployment stop
-
-Stop a deployment (`down` is a legacy alias):
+Stop a deployment:
 ```
-$ laconic-so deployment --dir <deployment-dir> stop
+$ laconic-so deployment --dir <deployment-dir> down
 ```
+Use `--delete-volumes` to also remove data volumes.
 
-Options:
-- `--delete-volumes` / `--preserve-volumes`: delete data volumes on stop (default: preserve)
-- `--skip-cluster-management` / `--perform-cluster-management`: skip kind cluster teardown (default: perform management). Use this to stop a single deployment without destroying a shared cluster.
-
-### deployment restart
-
-Restart a deployment with GitOps-aware workflow. Pulls latest stack code, syncs the deployment directory from the git-tracked spec, and restarts services:
-```
-$ laconic-so deployment --dir <deployment-dir> restart
-```
-
-See [deployment_patterns.md](deployment_patterns.md) for the recommended GitOps workflow.
-
-### deployment ps
+### deploy ps
 
 Show running services:
 ```
 $ laconic-so deployment --dir <deployment-dir> ps
 ```
 
-### deployment logs
+### deploy logs
 
 View service logs:
 ```
 $ laconic-so deployment --dir <deployment-dir> logs
 ```
 Use `-f` to follow and `-n <count>` to tail.
-
-### deployment exec
-
-Execute a command in a running service container:
-```
-$ laconic-so deployment --dir <deployment-dir> exec <service-name> "<command>"
-```
-
-### deployment status
-
-Show deployment status:
-```
-$ laconic-so deployment --dir <deployment-dir> status
-```
-
-### deployment port
-
-Show mapped ports for a service:
-```
-$ laconic-so deployment --dir <deployment-dir> port <service-name> <port>
-```
-
-### deployment push-images
-
-Push deployment images to a registry:
-```
-$ laconic-so deployment --dir <deployment-dir> push-images
-```
-
-### deployment run-job
-
-Run a one-time job in the deployment:
-```
-$ laconic-so deployment --dir <deployment-dir> run-job <job-name>
-```

docs/deployment_patterns.md

@@ -30,7 +30,7 @@ git commit -m "Add my-stack deployment configuration"
 git push
 
 # On deployment server: deploy from git-tracked spec
-laconic-so --stack my-stack deploy create \
+laconic-so deploy create \
   --spec-file /path/to/operator-repo/spec.yml \
   --deployment-dir my-deployment
 

pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
     "click>=8.1.6",
     "PyYAML>=6.0.1",
     "ruamel.yaml>=0.17.32",
-    "pydantic==1.10.9",
+    "pydantic==1.10.13",
     "tomli==2.0.1",
     "validators==0.22.0",
     "kubernetes>=28.1.0",

requirements.txt

@@ -6,7 +6,7 @@ python-on-whales>=0.64.0
 click>=8.1.6
 PyYAML>=6.0.1
 ruamel.yaml>=0.17.32
-pydantic==1.10.9
+pydantic==1.10.13
 tomli==2.0.1
 validators==0.22.0
 kubernetes>=28.1.0

setup.py

@@ -1,12 +1,12 @@
 # See
 # https://medium.com/nerd-for-tech/how-to-build-and-distribute-a-cli-tool-with-python-537ae41d9d78
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
-with open("README.md", "r", encoding="utf-8") as fh:
+with open("README.md", encoding="utf-8") as fh:
     long_description = fh.read()
-with open("requirements.txt", "r", encoding="utf-8") as fh:
+with open("requirements.txt", encoding="utf-8") as fh:
     requirements = fh.read()
-with open("stack_orchestrator/data/version.txt", "r", encoding="utf-8") as fh:
+with open("stack_orchestrator/data/version.txt", encoding="utf-8") as fh:
     version = fh.readlines()[-1].strip(" \n")
 setup(
     name="laconic-stack-orchestrator",

stack_orchestrator/base.py

@@ -15,9 +15,11 @@
 
 import os
 from abc import ABC, abstractmethod
-from stack_orchestrator.deploy.deploy import get_stack_status
+
 from decouple import config
 
+from stack_orchestrator.deploy.deploy import get_stack_status
+
 
 def get_stack(config, stack):
     if stack == "package-registry":

stack_orchestrator/build/build_containers.py

@@ -22,17 +22,19 @@
 # allow re-build of either all or specific containers
 
 import os
-import sys
-from decouple import config
 import subprocess
-import click
+import sys
 from pathlib import Path
-from stack_orchestrator.opts import opts
-from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
+
+import click
+from decouple import config
+
 from stack_orchestrator.base import get_npm_registry_url
 from stack_orchestrator.build.build_types import BuildContext
-from stack_orchestrator.build.publish import publish_image
 from stack_orchestrator.build.build_util import get_containers_in_scope
+from stack_orchestrator.build.publish import publish_image
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import error_exit, include_exclude_check, stack_is_external
 
 # TODO: find a place for this
 #    epilog="Config provided either in .env or settings.ini or env vars:
@@ -59,9 +61,7 @@ def make_container_build_env(
     container_build_env.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
     container_build_env.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {})
     container_build_env.update(
-        {"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args}
-        if extra_build_args
-        else {}
+        {"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {}
     )
     docker_host_env = os.getenv("DOCKER_HOST")
     if docker_host_env:
@@ -81,12 +81,8 @@ def process_container(build_context: BuildContext) -> bool:
 
     # Check if this is in an external stack
     if stack_is_external(build_context.stack):
-        container_parent_dir = Path(build_context.stack).parent.parent.joinpath(
-            "container-build"
-        )
-        temp_build_dir = container_parent_dir.joinpath(
-            build_context.container.replace("/", "-")
-        )
+        container_parent_dir = Path(build_context.stack).parent.parent.joinpath("container-build")
+        temp_build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
         temp_build_script_filename = temp_build_dir.joinpath("build.sh")
         # Now check if the container exists in the external stack.
         if not temp_build_script_filename.exists():
@@ -104,18 +100,13 @@ def process_container(build_context: BuildContext) -> bool:
         build_command = build_script_filename.as_posix()
     else:
         if opts.o.verbose:
-            print(
-                f"No script file found: {build_script_filename}, "
-                "using default build script"
-            )
+            print(f"No script file found: {build_script_filename}, " "using default build script")
         repo_dir = build_context.container.split("/")[1]
         # TODO: make this less of a hack -- should be specified in
         # some metadata somewhere. Check if we have a repo for this
         # container. If not, set the context dir to container-build subdir
         repo_full_path = os.path.join(build_context.dev_root_path, repo_dir)
-        repo_dir_or_build_dir = (
-            repo_full_path if os.path.exists(repo_full_path) else build_dir
-        )
+        repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
         build_command = (
             os.path.join(build_context.container_build_dir, "default-build.sh")
             + f" {default_container_tag} {repo_dir_or_build_dir}"
@@ -159,9 +150,7 @@ def process_container(build_context: BuildContext) -> bool:
     default=False,
     help="Publish the built images in the specified image registry",
 )
-@click.option(
-    "--image-registry", help="Specify the image registry for --publish-images"
-)
+@click.option("--image-registry", help="Specify the image registry for --publish-images")
 @click.pass_context
 def command(
     ctx,
@@ -185,14 +174,9 @@ def command(
 
     if local_stack:
         dev_root_path = os.getcwd()[0 : os.getcwd().rindex("stack-orchestrator")]
-        print(
-            f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: "
-            f"{dev_root_path}"
-        )
+        print(f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: " f"{dev_root_path}")
     else:
-        dev_root_path = os.path.expanduser(
-            config("CERC_REPO_BASE_DIR", default="~/cerc")
-        )
+        dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
 
     if not opts.o.quiet:
         print(f"Dev Root is: {dev_root_path}")
@@ -230,10 +214,7 @@ def command(
             else:
                 print(f"Error running build for {build_context.container}")
                 if not opts.o.continue_on_error:
-                    error_exit(
-                        "container build failed and --continue-on-error "
-                        "not set, exiting"
-                    )
+                    error_exit("container build failed and --continue-on-error " "not set, exiting")
                     sys.exit(1)
                 else:
                     print(

stack_orchestrator/build/build_npms.py

@@ -18,15 +18,17 @@
 # env vars:
 # CERC_REPO_BASE_DIR defaults to ~/cerc
 
+import importlib.resources
 import os
 import sys
-from shutil import rmtree, copytree
-from decouple import config
+from shutil import copytree, rmtree
+
 import click
-import importlib.resources
-from python_on_whales import docker, DockerException
+from decouple import config
+from python_on_whales import DockerException, docker
+
 from stack_orchestrator.base import get_stack
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config
+from stack_orchestrator.util import get_parsed_stack_config, include_exclude_check
 
 builder_js_image_name = "cerc/builder-js:local"
 
@@ -70,14 +72,9 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
 
     if local_stack:
         dev_root_path = os.getcwd()[0 : os.getcwd().rindex("stack-orchestrator")]
-        print(
-            f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: "
-            f"{dev_root_path}"
-        )
+        print(f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: " f"{dev_root_path}")
     else:
-        dev_root_path = os.path.expanduser(
-            config("CERC_REPO_BASE_DIR", default="~/cerc")
-        )
+        dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
 
     build_root_path = os.path.join(dev_root_path, "build-trees")
 
@@ -94,9 +91,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
     # See: https://stackoverflow.com/a/20885799/1701505
     from stack_orchestrator import data
 
-    with importlib.resources.open_text(
-        data, "npm-package-list.txt"
-    ) as package_list_file:
+    with importlib.resources.open_text(data, "npm-package-list.txt") as package_list_file:
         all_packages = package_list_file.read().splitlines()
 
     packages_in_scope = []
@@ -132,8 +127,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
         build_command = [
             "sh",
             "-c",
-            "cd /workspace && "
-            f"build-npm-package-local-dependencies.sh {npm_registry_url}",
+            "cd /workspace && " f"build-npm-package-local-dependencies.sh {npm_registry_url}",
         ]
         if not dry_run:
             if verbose:
@@ -151,9 +145,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
             envs.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
             envs.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {})
             envs.update(
-                {"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args}
-                if extra_build_args
-                else {}
+                {"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {}
             )
             try:
                 docker.run(
@@ -176,16 +168,10 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
             except DockerException as e:
                 print(f"Error executing build for {package} in container:\n {e}")
                 if not continue_on_error:
-                    print(
-                        "FATAL Error: build failed and --continue-on-error "
-                        "not set, exiting"
-                    )
+                    print("FATAL Error: build failed and --continue-on-error " "not set, exiting")
                     sys.exit(1)
                 else:
-                    print(
-                        "****** Build Error, continuing because "
-                        "--continue-on-error is set"
-                    )
+                    print("****** Build Error, continuing because " "--continue-on-error is set")
 
         else:
             print("Skipped")
@@ -203,10 +189,7 @@ def _ensure_prerequisites():
     # Tell the user how to build it if not
     images = docker.image.list(builder_js_image_name)
     if len(images) == 0:
-        print(
-            f"FATAL: builder image: {builder_js_image_name} is required "
-            "but was not found"
-        )
+        print(f"FATAL: builder image: {builder_js_image_name} is required " "but was not found")
         print(
             "Please run this command to create it: "
             "laconic-so --stack build-support build-containers"

stack_orchestrator/build/build_types.py

@@ -16,7 +16,6 @@
 
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Mapping
 
 
 @dataclass
@@ -24,5 +23,5 @@ class BuildContext:
     stack: str
     container: str
     container_build_dir: Path
-    container_build_env: Mapping[str, str]
+    container_build_env: dict[str, str]
     dev_root_path: str

stack_orchestrator/build/build_util.py

@@ -30,9 +30,7 @@ def get_containers_in_scope(stack: str):
         # See: https://stackoverflow.com/a/20885799/1701505
         from stack_orchestrator import data
 
-        with importlib.resources.open_text(
-            data, "container-image-list.txt"
-        ) as container_list_file:
+        with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
             containers_in_scope = container_list_file.read().splitlines()
 
     if opts.o.verbose:

stack_orchestrator/build/build_webapp.py

@@ -23,20 +23,19 @@
 
 import os
 import sys
-
-from decouple import config
-import click
 from pathlib import Path
+
+import click
+from decouple import config
+
 from stack_orchestrator.build import build_containers
-from stack_orchestrator.deploy.webapp.util import determine_base_container, TimedLogger
 from stack_orchestrator.build.build_types import BuildContext
+from stack_orchestrator.deploy.webapp.util import TimedLogger, determine_base_container
 
 
 @click.command()
 @click.option("--base-container")
-@click.option(
-    "--source-repo", help="directory containing the webapp to build", required=True
-)
+@click.option("--source-repo", help="directory containing the webapp to build", required=True)
 @click.option(
     "--force-rebuild",
     is_flag=True,
@@ -64,13 +63,10 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     if local_stack:
         dev_root_path = os.getcwd()[0 : os.getcwd().rindex("stack-orchestrator")]
         logger.log(
-            f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: "
-            f"{dev_root_path}"
+            f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: " f"{dev_root_path}"
         )
     else:
-        dev_root_path = os.path.expanduser(
-            config("CERC_REPO_BASE_DIR", default="~/cerc")
-        )
+        dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
 
     if verbose:
         logger.log(f"Dev Root is: {dev_root_path}")

stack_orchestrator/build/fetch_containers.py

@@ -13,19 +13,19 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-import click
-from dataclasses import dataclass
 import json
 import platform
+from dataclasses import dataclass
+
+import click
+import requests
 from python_on_whales import DockerClient
 from python_on_whales.components.manifest.cli_wrapper import ManifestCLI, ManifestList
 from python_on_whales.utils import run
-import requests
-from typing import List
 
-from stack_orchestrator.opts import opts
-from stack_orchestrator.util import include_exclude_check, error_exit
 from stack_orchestrator.build.build_util import get_containers_in_scope
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import error_exit, include_exclude_check
 
 # Experimental fetch-container command
 
@@ -55,7 +55,7 @@ def _local_tag_for(container: str):
 # $ curl -u "my-username:my-token" -X GET \
 #   "https://<container-registry-hostname>/v2/cerc-io/cerc/test-container/tags/list"
 # {"name":"cerc-io/cerc/test-container","tags":["202402232130","202402232208"]}
-def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List[str]:
+def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> list[str]:
     # registry looks like: git.vdb.to/cerc-io
     registry_parts = registry_info.registry.split("/")
     url = f"https://{registry_parts[0]}/v2/{registry_parts[1]}/{container}/tags/list"
@@ -68,16 +68,15 @@ def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List
         tag_info = response.json()
         if opts.o.debug:
             print(f"container tags list: {tag_info}")
-        tags_array = tag_info["tags"]
+        tags_array: list[str] = tag_info["tags"]
         return tags_array
     else:
         error_exit(
-            f"failed to fetch tags from image registry, "
-            f"status code: {response.status_code}"
+            f"failed to fetch tags from image registry, " f"status code: {response.status_code}"
         )
 
 
-def _find_latest(candidate_tags: List[str]):
+def _find_latest(candidate_tags: list[str]):
     # Lex sort should give us the latest first
     sorted_candidates = sorted(candidate_tags)
     if opts.o.debug:
@@ -86,8 +85,8 @@ def _find_latest(candidate_tags: List[str]):
 
 
 def _filter_for_platform(
-    container: str, registry_info: RegistryInfo, tag_list: List[str]
-) -> List[str]:
+    container: str, registry_info: RegistryInfo, tag_list: list[str]
+) -> list[str]:
     filtered_tags = []
     this_machine = platform.machine()
     # Translate between Python and docker platform names
@@ -151,15 +150,9 @@ def _add_local_tag(remote_tag: str, registry: str, local_tag: str):
     default=False,
     help="Overwrite a locally built image, if present",
 )
-@click.option(
-    "--image-registry", required=True, help="Specify the image registry to fetch from"
-)
-@click.option(
-    "--registry-username", required=True, help="Specify the image registry username"
-)
-@click.option(
-    "--registry-token", required=True, help="Specify the image registry access token"
-)
+@click.option("--image-registry", required=True, help="Specify the image registry to fetch from")
+@click.option("--registry-username", required=True, help="Specify the image registry username")
+@click.option("--registry-token", required=True, help="Specify the image registry access token")
 @click.pass_context
 def command(
     ctx,

stack_orchestrator/build/publish.py

@@ -14,6 +14,7 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from datetime import datetime
+
 from python_on_whales import DockerClient
 
 from stack_orchestrator.opts import opts

stack_orchestrator/constants.py

@@ -29,7 +29,6 @@ network_key = "network"
 http_proxy_key = "http-proxy"
 image_registry_key = "image-registry"
 configmaps_key = "configmaps"
-secrets_key = "secrets"
 resources_key = "resources"
 volumes_key = "volumes"
 security_key = "security"

stack_orchestrator/data/compose-jobs/docker-compose-test-job.yml

@@ -1,5 +0,0 @@
-services:
-  test-job:
-    image: cerc/test-container:local
-    entrypoint: /bin/sh
-    command: ["-c", "echo 'Job completed successfully'"]

stack_orchestrator/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py

@@ -2,12 +2,11 @@
 
 import argparse
 import os
+import random
 import sys
+from subprocess import Popen
 
 import psycopg
-import random
-
-from subprocess import Popen
 from fabric import Connection
 
 
@@ -27,27 +26,19 @@ def dump_src_db_to_file(db_host, db_port, db_user, db_password, db_name, file_na
 def establish_ssh_tunnel(ssh_host, ssh_port, ssh_user, db_host, db_port):
     local_port = random.randint(11000, 12000)
     conn = Connection(host=ssh_host, port=ssh_port, user=ssh_user)
-    fw = conn.forward_local(
-        local_port=local_port, remote_port=db_port, remote_host=db_host
-    )
+    fw = conn.forward_local(local_port=local_port, remote_port=db_port, remote_host=db_host)
     return conn, fw, local_port
 
 
 def load_db_from_file(db_host, db_port, db_user, db_password, db_name, file_name):
-    connstr = "host=%s port=%s user=%s password=%s sslmode=disable dbname=%s" % (
-        db_host,
-        db_port,
-        db_user,
-        db_password,
-        db_name,
-    )
+    connstr = f"host={db_host} port={db_port} user={db_user} password={db_password} sslmode=disable dbname={db_name}"
     with psycopg.connect(connstr) as conn:
         with conn.cursor() as cur:
             print(
                 f"Importing from {file_name} to {db_host}:{db_port}/{db_name}... ",
                 end="",
             )
-            cur.execute(open(file_name, "rt").read())
+            cur.execute(open(file_name).read())
             print("DONE")
 
 
@@ -60,9 +51,7 @@ if __name__ == "__main__":
     parser.add_argument("--src-dbpw", help="DB password", required=True)
     parser.add_argument("--src-dbname", help="dbname", default="keycloak")
 
-    parser.add_argument(
-        "--dst-file", help="Destination filename", default="keycloak-mirror.sql"
-    )
+    parser.add_argument("--dst-file", help="Destination filename", default="keycloak-mirror.sql")
 
     parser.add_argument("--live-import", help="run the import", action="store_true")
 

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/accounts/mnemonic_to_csv.py

@@ -1,7 +1,8 @@
-from web3.auto import w3
-import ruamel.yaml as yaml
 import sys
 
+import ruamel.yaml as yaml
+from web3.auto import w3
+
 w3.eth.account.enable_unaudited_hdwallet_features()
 
 testnet_config_path = "genesis-config.yaml"
@@ -11,8 +12,6 @@ if len(sys.argv) > 1:
 with open(testnet_config_path) as stream:
     data = yaml.safe_load(stream)
 
-for key, value in data["el_premine"].items():
-    acct = w3.eth.account.from_mnemonic(
-        data["mnemonic"], account_path=key, passphrase=""
-    )
-    print("%s,%s,%s" % (key, acct.address, acct.key.hex()))
+for key, _value in data["el_premine"].items():
+    acct = w3.eth.account.from_mnemonic(data["mnemonic"], account_path=key, passphrase="")
+    print(f"{key},{acct.address},{acct.key.hex()}")

stack_orchestrator/data/stacks/mainnet-blast/deploy/commands.py

@@ -16,13 +16,14 @@
 
 from pathlib import Path
 from shutil import copy
+
 import yaml
 
 
 def create(context, extra_args):
     # Our goal here is just to copy the json files for blast
     yml_path = context.deployment_dir.joinpath("spec.yml")
-    with open(yml_path, "r") as file:
+    with open(yml_path) as file:
         data = yaml.safe_load(file)
 
     mount_point = data["volumes"]["blast-data"]

stack_orchestrator/data/stacks/mainnet-eth/deploy/commands.py

@@ -27,8 +27,6 @@ def setup(ctx):
 def create(ctx, extra_args):
     # Generate the JWT secret and save to its config file
     secret = token_hex(32)
-    jwt_file_path = ctx.deployment_dir.joinpath(
-        "data", "mainnet_eth_config_data", "jwtsecret"
-    )
+    jwt_file_path = ctx.deployment_dir.joinpath("data", "mainnet_eth_config_data", "jwtsecret")
     with open(jwt_file_path, "w+") as jwt_file:
         jwt_file.write(secret)

stack_orchestrator/data/stacks/mainnet-laconic/deploy/commands.py

@@ -13,22 +13,23 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from stack_orchestrator.util import get_yaml
+import os
+import re
+import sys
+from enum import Enum
+from pathlib import Path
+from shutil import copyfile, copytree
+
+import tomli
 from stack_orchestrator.deploy.deploy_types import (
     DeployCommandContext,
     LaconicStackSetupCommand,
 )
+from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
 from stack_orchestrator.deploy.stack_state import State
-from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command
 from stack_orchestrator.opts import opts
-from enum import Enum
-from pathlib import Path
-from shutil import copyfile, copytree
-import os
-import sys
-import tomli
-import re
+from stack_orchestrator.util import get_yaml
 
 default_spec_file_content = ""
 
@@ -80,9 +81,7 @@ def _copy_gentx_files(network_dir: Path, gentx_file_list: str):
         gentx_file_path = Path(gentx_file)
         copyfile(
             gentx_file_path,
-            os.path.join(
-                network_dir, "config", "gentx", os.path.basename(gentx_file_path)
-            ),
+            os.path.join(network_dir, "config", "gentx", os.path.basename(gentx_file_path)),
         )
 
 
@@ -91,7 +90,7 @@ def _remove_persistent_peers(network_dir: Path):
     if not config_file_path.exists():
         print("Error: config.toml not found")
         sys.exit(1)
-    with open(config_file_path, "r") as input_file:
+    with open(config_file_path) as input_file:
         config_file_content = input_file.read()
         persistent_peers_pattern = '^persistent_peers = "(.+?)"'
         replace_with = 'persistent_peers = ""'
@@ -110,7 +109,7 @@ def _insert_persistent_peers(config_dir: Path, new_persistent_peers: str):
     if not config_file_path.exists():
         print("Error: config.toml not found")
         sys.exit(1)
-    with open(config_file_path, "r") as input_file:
+    with open(config_file_path) as input_file:
         config_file_content = input_file.read()
         persistent_peers_pattern = r'^persistent_peers = ""'
         replace_with = f'persistent_peers = "{new_persistent_peers}"'
@@ -129,7 +128,7 @@ def _enable_cors(config_dir: Path):
     if not config_file_path.exists():
         print("Error: config.toml not found")
         sys.exit(1)
-    with open(config_file_path, "r") as input_file:
+    with open(config_file_path) as input_file:
         config_file_content = input_file.read()
         cors_pattern = r"^cors_allowed_origins = \[]"
         replace_with = 'cors_allowed_origins = ["*"]'
@@ -142,13 +141,11 @@ def _enable_cors(config_dir: Path):
     if not app_file_path.exists():
         print("Error: app.toml not found")
         sys.exit(1)
-    with open(app_file_path, "r") as input_file:
+    with open(app_file_path) as input_file:
         app_file_content = input_file.read()
         cors_pattern = r"^enabled-unsafe-cors = false"
         replace_with = "enabled-unsafe-cors = true"
-        app_file_content = re.sub(
-            cors_pattern, replace_with, app_file_content, flags=re.MULTILINE
-        )
+        app_file_content = re.sub(cors_pattern, replace_with, app_file_content, flags=re.MULTILINE)
     with open(app_file_path, "w") as output_file:
         output_file.write(app_file_content)
 
@@ -158,7 +155,7 @@ def _set_listen_address(config_dir: Path):
     if not config_file_path.exists():
         print("Error: config.toml not found")
         sys.exit(1)
-    with open(config_file_path, "r") as input_file:
+    with open(config_file_path) as input_file:
         config_file_content = input_file.read()
         existing_pattern = r'^laddr = "tcp://127.0.0.1:26657"'
         replace_with = 'laddr = "tcp://0.0.0.0:26657"'
@@ -172,7 +169,7 @@ def _set_listen_address(config_dir: Path):
     if not app_file_path.exists():
         print("Error: app.toml not found")
         sys.exit(1)
-    with open(app_file_path, "r") as input_file:
+    with open(app_file_path) as input_file:
         app_file_content = input_file.read()
         existing_pattern1 = r'^address = "tcp://localhost:1317"'
         replace_with1 = 'address = "tcp://0.0.0.0:1317"'
@@ -192,10 +189,7 @@ def _phase_from_params(parameters):
     phase = SetupPhase.ILLEGAL
     if parameters.initialize_network:
         if parameters.join_network or parameters.create_network:
-            print(
-                "Can't supply --join-network or --create-network "
-                "with --initialize-network"
-            )
+            print("Can't supply --join-network or --create-network " "with --initialize-network")
             sys.exit(1)
         if not parameters.chain_id:
             print("--chain-id is required")
@@ -207,26 +201,17 @@ def _phase_from_params(parameters):
         phase = SetupPhase.INITIALIZE
     elif parameters.join_network:
         if parameters.initialize_network or parameters.create_network:
-            print(
-                "Can't supply --initialize-network or --create-network "
-                "with --join-network"
-            )
+            print("Can't supply --initialize-network or --create-network " "with --join-network")
             sys.exit(1)
         phase = SetupPhase.JOIN
     elif parameters.create_network:
         if parameters.initialize_network or parameters.join_network:
-            print(
-                "Can't supply --initialize-network or --join-network "
-                "with --create-network"
-            )
+            print("Can't supply --initialize-network or --join-network " "with --create-network")
             sys.exit(1)
         phase = SetupPhase.CREATE
     elif parameters.connect_network:
         if parameters.initialize_network or parameters.join_network:
-            print(
-                "Can't supply --initialize-network or --join-network "
-                "with --connect-network"
-            )
+            print("Can't supply --initialize-network or --join-network " "with --connect-network")
             sys.exit(1)
         phase = SetupPhase.CONNECT
     return phase
@@ -341,8 +326,7 @@ def setup(
         output3, status3 = run_container_command(
             command_context,
             "laconicd",
-            f"laconicd cometbft show-validator "
-            f"--home {laconicd_home_path_in_container}",
+            f"laconicd cometbft show-validator " f"--home {laconicd_home_path_in_container}",
             mounts,
         )
         print(f"Node validator address: {output3}")
@@ -361,23 +345,16 @@ def setup(
             # Copy it into our network dir
             genesis_file_path = Path(parameters.genesis_file)
             if not os.path.exists(genesis_file_path):
-                print(
-                    f"Error: supplied genesis file: {parameters.genesis_file} "
-                    "does not exist."
-                )
+                print(f"Error: supplied genesis file: {parameters.genesis_file} " "does not exist.")
                 sys.exit(1)
             copyfile(
                 genesis_file_path,
-                os.path.join(
-                    network_dir, "config", os.path.basename(genesis_file_path)
-                ),
+                os.path.join(network_dir, "config", os.path.basename(genesis_file_path)),
             )
         else:
             # We're generating the genesis file
             # First look in the supplied gentx files for the other nodes' keys
-            other_node_keys = _get_node_keys_from_gentx_files(
-                parameters.gentx_address_list
-            )
+            other_node_keys = _get_node_keys_from_gentx_files(parameters.gentx_address_list)
             # Add those keys to our genesis, with balances we determine here (why?)
             outputk = None
             for other_node_key in other_node_keys:
@@ -398,8 +375,7 @@ def setup(
             output1, status1 = run_container_command(
                 command_context,
                 "laconicd",
-                f"laconicd genesis collect-gentxs "
-                f"--home {laconicd_home_path_in_container}",
+                f"laconicd genesis collect-gentxs " f"--home {laconicd_home_path_in_container}",
                 mounts,
             )
             if options.debug:
@@ -416,8 +392,7 @@ def setup(
         output2, status1 = run_container_command(
             command_context,
             "laconicd",
-            f"laconicd genesis validate-genesis "
-            f"--home {laconicd_home_path_in_container}",
+            f"laconicd genesis validate-genesis " f"--home {laconicd_home_path_in_container}",
             mounts,
         )
         print(f"validate-genesis result: {output2}")
@@ -452,9 +427,7 @@ def create(deployment_context: DeploymentContext, extra_args):
         sys.exit(1)
     # Copy the network directory contents into our deployment
     # TODO: change this to work with non local paths
-    deployment_config_dir = deployment_context.deployment_dir.joinpath(
-        "data", "laconicd-config"
-    )
+    deployment_config_dir = deployment_context.deployment_dir.joinpath("data", "laconicd-config")
     copytree(config_dir_path, deployment_config_dir, dirs_exist_ok=True)
     # If supplied, add the initial persistent peers to the config file
     if extra_args[1]:
@@ -465,9 +438,7 @@ def create(deployment_context: DeploymentContext, extra_args):
     _set_listen_address(deployment_config_dir)
     # Copy the data directory contents into our deployment
     # TODO: change this to work with non local paths
-    deployment_data_dir = deployment_context.deployment_dir.joinpath(
-        "data", "laconicd-data"
-    )
+    deployment_data_dir = deployment_context.deployment_dir.joinpath("data", "laconicd-data")
     copytree(data_dir_path, deployment_data_dir, dirs_exist_ok=True)
 
 

stack_orchestrator/data/stacks/test/deploy/commands.py

@@ -13,15 +13,16 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from stack_orchestrator.util import get_yaml
-from stack_orchestrator.deploy.deploy_types import DeployCommandContext
-from stack_orchestrator.deploy.deployment_context import DeploymentContext
-from stack_orchestrator.deploy.stack_state import State
-from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command
 from pathlib import Path
 
+from stack_orchestrator.deploy.deploy_types import DeployCommandContext
+from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command
+from stack_orchestrator.deploy.deployment_context import DeploymentContext
+from stack_orchestrator.deploy.stack_state import State
+from stack_orchestrator.util import get_yaml
+
 default_spec_file_content = """config:
-    test_variable_1: test-value-1
+    test-variable-1: test-value-1
 """
 
 

stack_orchestrator/data/stacks/test/stack.yml

@@ -7,5 +7,3 @@ containers:
   - cerc/test-container
 pods:
   - test
-jobs:
-  - test-job

stack_orchestrator/deploy/compose/deploy_docker.py

@@ -14,12 +14,13 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from pathlib import Path
-from typing import Optional
+
 from python_on_whales import DockerClient, DockerException
+
 from stack_orchestrator.deploy.deployer import (
     Deployer,
-    DeployerException,
     DeployerConfigGenerator,
+    DeployerException,
 )
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
 from stack_orchestrator.opts import opts
@@ -32,10 +33,10 @@ class DockerDeployer(Deployer):
     def __init__(
         self,
         type: str,
-        deployment_context: Optional[DeploymentContext],
+        deployment_context: DeploymentContext | None,
         compose_files: list,
-        compose_project_name: Optional[str],
-        compose_env_file: Optional[str],
+        compose_project_name: str | None,
+        compose_env_file: str | None,
     ) -> None:
         self.docker = DockerClient(
             compose_files=compose_files,
@@ -53,21 +54,21 @@ class DockerDeployer(Deployer):
             try:
                 return self.docker.compose.up(detach=detach, services=services)
             except DockerException as e:
-                raise DeployerException(e)
+                raise DeployerException(e) from e
 
     def down(self, timeout, volumes, skip_cluster_management):
         if not opts.o.dry_run:
             try:
                 return self.docker.compose.down(timeout=timeout, volumes=volumes)
             except DockerException as e:
-                raise DeployerException(e)
+                raise DeployerException(e) from e
 
-    def update(self):
+    def update_envs(self):
         if not opts.o.dry_run:
             try:
                 return self.docker.compose.restart()
             except DockerException as e:
-                raise DeployerException(e)
+                raise DeployerException(e) from e
 
     def status(self):
         if not opts.o.dry_run:
@@ -75,23 +76,21 @@ class DockerDeployer(Deployer):
                 for p in self.docker.compose.ps():
                     print(f"{p.name}\t{p.state.status}")
             except DockerException as e:
-                raise DeployerException(e)
+                raise DeployerException(e) from e
 
     def ps(self):
         if not opts.o.dry_run:
             try:
                 return self.docker.compose.ps()
             except DockerException as e:
-                raise DeployerException(e)
+                raise DeployerException(e) from e
 
     def port(self, service, private_port):
         if not opts.o.dry_run:
             try:
-                return self.docker.compose.port(
-                    service=service, private_port=private_port
-                )
+                return self.docker.compose.port(service=service, private_port=private_port)
             except DockerException as e:
-                raise DeployerException(e)
+                raise DeployerException(e) from e
 
     def execute(self, service, command, tty, envs):
         if not opts.o.dry_run:
@@ -100,7 +99,7 @@ class DockerDeployer(Deployer):
                     service=service, command=command, tty=tty, envs=envs
                 )
             except DockerException as e:
-                raise DeployerException(e)
+                raise DeployerException(e) from e
 
     def logs(self, services, tail, follow, stream):
         if not opts.o.dry_run:
@@ -109,7 +108,7 @@ class DockerDeployer(Deployer):
                     services=services, tail=tail, follow=follow, stream=stream
                 )
             except DockerException as e:
-                raise DeployerException(e)
+                raise DeployerException(e) from e
 
     def run(
         self,
@@ -118,10 +117,14 @@ class DockerDeployer(Deployer):
         user=None,
         volumes=None,
         entrypoint=None,
-        env={},
-        ports=[],
+        env=None,
+        ports=None,
         detach=False,
     ):
+        if ports is None:
+            ports = []
+        if env is None:
+            env = {}
         if not opts.o.dry_run:
             try:
                 return self.docker.run(
@@ -136,9 +139,9 @@ class DockerDeployer(Deployer):
                     publish_all=len(ports) == 0,
                 )
             except DockerException as e:
-                raise DeployerException(e)
+                raise DeployerException(e) from e
 
-    def run_job(self, job_name: str, release_name: Optional[str] = None):
+    def run_job(self, job_name: str, release_name: str | None = None):
         # release_name is ignored for Docker deployments (only used for K8s/Helm)
         if not opts.o.dry_run:
             try:
@@ -155,9 +158,7 @@ class DockerDeployer(Deployer):
                 )
 
                 if not job_compose_file.exists():
-                    raise DeployerException(
-                        f"Job compose file not found: {job_compose_file}"
-                    )
+                    raise DeployerException(f"Job compose file not found: {job_compose_file}")
 
                 if opts.o.verbose:
                     print(f"Running job from: {job_compose_file}")
@@ -175,7 +176,7 @@ class DockerDeployer(Deployer):
                 return job_docker.compose.run(service=job_name, remove=True, tty=True)
 
             except DockerException as e:
-                raise DeployerException(e)
+                raise DeployerException(e) from e
 
 
 class DockerDeployerConfigGenerator(DeployerConfigGenerator):

stack_orchestrator/deploy/deploy.py

@@ -15,37 +15,37 @@
 
 # Deploys the system components using a deployer (either docker-compose or k8s)
 
-import hashlib
 import copy
+import hashlib
 import os
+import subprocess
 import sys
 from dataclasses import dataclass
 from importlib import resources
-from typing import Optional
-import subprocess
-import click
 from pathlib import Path
+
+import click
+
 from stack_orchestrator import constants
-from stack_orchestrator.opts import opts
-from stack_orchestrator.util import (
-    get_stack_path,
-    include_exclude_check,
-    get_parsed_stack_config,
-    global_options2,
-    get_dev_root_path,
-    stack_is_in_deployment,
-    resolve_compose_file,
-    get_job_list,
-)
-from stack_orchestrator.deploy.deployer import DeployerException
-from stack_orchestrator.deploy.deployer_factory import getDeployer
 from stack_orchestrator.deploy.compose.deploy_docker import DockerDeployer
 from stack_orchestrator.deploy.deploy_types import ClusterContext, DeployCommandContext
+from stack_orchestrator.deploy.deployer import DeployerException
+from stack_orchestrator.deploy.deployer_factory import getDeployer
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
 from stack_orchestrator.deploy.deployment_create import create as deployment_create
 from stack_orchestrator.deploy.deployment_create import init as deployment_init
 from stack_orchestrator.deploy.deployment_create import setup as deployment_setup
 from stack_orchestrator.deploy.k8s import k8s_command
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import (
+    get_dev_root_path,
+    get_parsed_stack_config,
+    get_stack_path,
+    global_options2,
+    include_exclude_check,
+    resolve_compose_file,
+    stack_is_in_deployment,
+)
 
 
 @click.group()
@@ -53,9 +53,7 @@ from stack_orchestrator.deploy.k8s import k8s_command
 @click.option("--exclude", help="don't start these components")
 @click.option("--env-file", help="env file to be used")
 @click.option("--cluster", help="specify a non-default cluster name")
-@click.option(
-    "--deploy-to", help="cluster system to deploy to (compose or k8s or k8s-kind)"
-)
+@click.option("--deploy-to", help="cluster system to deploy to (compose or k8s or k8s-kind)")
 @click.pass_context
 def command(ctx, include, exclude, env_file, cluster, deploy_to):
     """deploy a stack"""
@@ -94,7 +92,7 @@ def command(ctx, include, exclude, env_file, cluster, deploy_to):
 
 def create_deploy_context(
     global_context,
-    deployment_context: Optional[DeploymentContext],
+    deployment_context: DeploymentContext | None,
     stack,
     include,
     exclude,
@@ -117,9 +115,7 @@ def create_deploy_context(
 
     # For helm chart deployments, skip compose file loading
     if is_helm_chart_deployment:
-        cluster_context = ClusterContext(
-            global_context, cluster, [], [], [], None, env_file
-        )
+        cluster_context = ClusterContext(global_context, cluster, [], [], [], None, env_file)
     else:
         cluster_context = _make_cluster_context(
             global_context, stack, include, exclude, cluster, env_file
@@ -131,14 +127,11 @@ def create_deploy_context(
         compose_files=cluster_context.compose_files,
         compose_project_name=cluster_context.cluster,
         compose_env_file=cluster_context.env_file,
-        job_compose_files=cluster_context.job_compose_files,
     )
     return DeployCommandContext(stack, cluster_context, deployer)
 
 
-def up_operation(
-    ctx, services_list, stay_attached=False, skip_cluster_management=False
-):
+def up_operation(ctx, services_list, stay_attached=False, skip_cluster_management=False):
     global_context = ctx.parent.parent.obj
     deploy_context = ctx.obj
     cluster_context = deploy_context.cluster_context
@@ -184,8 +177,14 @@ def status_operation(ctx):
     ctx.obj.deployer.status()
 
 
-def update_operation(ctx):
-    ctx.obj.deployer.update()
+def prepare_operation(ctx, skip_cluster_management=False):
+    ctx.obj.deployer.prepare(
+        skip_cluster_management=skip_cluster_management,
+    )
+
+
+def update_envs_operation(ctx):
+    ctx.obj.deployer.update_envs()
 
 
 def ps_operation(ctx):
@@ -205,8 +204,7 @@ def ps_operation(ctx):
                         print(f"{port_mapping}", end="")
                     else:
                         print(
-                            f"{mapping[0]['HostIp']}:{mapping[0]['HostPort']}"
-                            f"->{port_mapping}",
+                            f"{mapping[0]['HostIp']}:{mapping[0]['HostPort']}" f"->{port_mapping}",
                             end="",
                         )
                     comma = ", "
@@ -256,11 +254,11 @@ def logs_operation(ctx, tail: int, follow: bool, extra_args: str):
     logs_stream = ctx.obj.deployer.logs(
         services=services_list, tail=tail, follow=follow, stream=True
     )
-    for stream_type, stream_content in logs_stream:
+    for _stream_type, stream_content in logs_stream:
         print(stream_content.decode("utf-8"), end="")
 
 
-def run_job_operation(ctx, job_name: str, helm_release: Optional[str] = None):
+def run_job_operation(ctx, job_name: str, helm_release: str | None = None):
     global_context = ctx.parent.parent.obj
     if not global_context.dry_run:
         print(f"Running job: {job_name}")
@@ -280,9 +278,7 @@ def up(ctx, extra_args):
 
 
 @command.command()
-@click.option(
-    "--delete-volumes/--preserve-volumes", default=False, help="delete data volumes"
-)
+@click.option("--delete-volumes/--preserve-volumes", default=False, help="delete data volumes")
 @click.argument("extra_args", nargs=-1)  # help: command: down<service1> <service2>
 @click.pass_context
 def down(ctx, delete_volumes, extra_args):
@@ -382,14 +378,10 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
     else:
         # See:
         # https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
-        compose_dir = (
-            Path(__file__).absolute().parent.parent.joinpath("data", "compose")
-        )
+        compose_dir = Path(__file__).absolute().parent.parent.joinpath("data", "compose")
 
     if cluster is None:
-        cluster = _make_default_cluster_name(
-            deployment, compose_dir, stack, include, exclude
-        )
+        cluster = _make_default_cluster_name(deployment, compose_dir, stack, include, exclude)
     else:
         _make_default_cluster_name(deployment, compose_dir, stack, include, exclude)
 
@@ -405,10 +397,8 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
         stack_config = get_parsed_stack_config(stack)
         if stack_config is not None:
             # TODO: syntax check the input here
-            pods_in_scope = stack_config.get("pods") or []
-            cluster_config = (
-                stack_config["config"] if "config" in stack_config else None
-            )
+            pods_in_scope = stack_config["pods"]
+            cluster_config = stack_config["config"] if "config" in stack_config else None
     else:
         pods_in_scope = all_pods
 
@@ -430,43 +420,29 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
         if include_exclude_check(pod_name, include, exclude):
             if pod_repository is None or pod_repository == "internal":
                 if deployment:
-                    compose_file_name = os.path.join(
-                        compose_dir, f"docker-compose-{pod_path}.yml"
-                    )
+                    compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml")
                 else:
                     compose_file_name = resolve_compose_file(stack, pod_name)
             else:
                 if deployment:
-                    compose_file_name = os.path.join(
-                        compose_dir, f"docker-compose-{pod_name}.yml"
-                    )
+                    compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml")
                     pod_pre_start_command = pod.get("pre_start_command")
                     pod_post_start_command = pod.get("post_start_command")
-                    script_dir = compose_dir.parent.joinpath(
-                        "pods", pod_name, "scripts"
-                    )
+                    script_dir = compose_dir.parent.joinpath("pods", pod_name, "scripts")
                     if pod_pre_start_command is not None:
-                        pre_start_commands.append(
-                            os.path.join(script_dir, pod_pre_start_command)
-                        )
+                        pre_start_commands.append(os.path.join(script_dir, pod_pre_start_command))
                     if pod_post_start_command is not None:
-                        post_start_commands.append(
-                            os.path.join(script_dir, pod_post_start_command)
-                        )
+                        post_start_commands.append(os.path.join(script_dir, pod_post_start_command))
                 else:
                     # TODO: fix this code for external stack with scripts
                     pod_root_dir = os.path.join(
                         dev_root_path, pod_repository.split("/")[-1], pod["path"]
                     )
-                    compose_file_name = os.path.join(
-                        pod_root_dir, f"docker-compose-{pod_name}.yml"
-                    )
+                    compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml")
                     pod_pre_start_command = pod.get("pre_start_command")
                     pod_post_start_command = pod.get("post_start_command")
                     if pod_pre_start_command is not None:
-                        pre_start_commands.append(
-                            os.path.join(pod_root_dir, pod_pre_start_command)
-                        )
+                        pre_start_commands.append(os.path.join(pod_root_dir, pod_pre_start_command))
                     if pod_post_start_command is not None:
                         post_start_commands.append(
                             os.path.join(pod_root_dir, pod_post_start_command)
@@ -479,22 +455,6 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
     if ctx.verbose:
         print(f"files: {compose_files}")
 
-    # Gather job compose files (from compose-jobs/ directory in deployment)
-    job_compose_files = []
-    if deployment and stack:
-        stack_config = get_parsed_stack_config(stack)
-        if stack_config:
-            jobs = get_job_list(stack_config)
-            compose_jobs_dir = stack.joinpath("compose-jobs")
-            for job in jobs:
-                job_file_name = os.path.join(
-                    compose_jobs_dir, f"docker-compose-{job}.yml"
-                )
-                if os.path.exists(job_file_name):
-                    job_compose_files.append(job_file_name)
-            if ctx.verbose:
-                print(f"job files: {job_compose_files}")
-
     return ClusterContext(
         ctx,
         cluster,
@@ -503,7 +463,6 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
         post_start_commands,
         cluster_config,
         env_file,
-        job_compose_files=job_compose_files if job_compose_files else None,
     )
 
 
@@ -527,9 +486,7 @@ def _run_command(ctx, cluster_name, command):
     command_env["CERC_SO_COMPOSE_PROJECT"] = cluster_name
     if ctx.debug:
         command_env["CERC_SCRIPT_DEBUG"] = "true"
-    command_result = subprocess.run(
-        command_file, shell=True, env=command_env, cwd=command_dir
-    )
+    command_result = subprocess.run(command_file, shell=True, env=command_env, cwd=command_dir)
     if command_result.returncode != 0:
         print(f"FATAL Error running command: {command}")
         sys.exit(1)
@@ -586,9 +543,7 @@ def _orchestrate_cluster_config(ctx, cluster_config, deployer, container_exec_en
                         # "It returned with code 1"
                         if "It returned with code 1" in str(error):
                             if ctx.verbose:
-                                print(
-                                    "Config export script returned an error, re-trying"
-                                )
+                                print("Config export script returned an error, re-trying")
                         # If the script failed to execute
                         # (e.g. the file is not there) then we get:
                         # "It returned with code 2"

stack_orchestrator/deploy/deploy_types.py

@@ -13,8 +13,9 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from typing import List, Mapping, Optional
+from collections.abc import Mapping
 from dataclasses import dataclass
+
 from stack_orchestrator.command_types import CommandOptions
 from stack_orchestrator.deploy.deployer import Deployer
 
@@ -23,20 +24,19 @@ from stack_orchestrator.deploy.deployer import Deployer
 class ClusterContext:
     # TODO: this should be in its own object not stuffed in here
     options: CommandOptions
-    cluster: Optional[str]
-    compose_files: List[str]
-    pre_start_commands: List[str]
-    post_start_commands: List[str]
-    config: Optional[str]
-    env_file: Optional[str]
-    job_compose_files: Optional[List[str]] = None
+    cluster: str | None
+    compose_files: list[str]
+    pre_start_commands: list[str]
+    post_start_commands: list[str]
+    config: str | None
+    env_file: str | None
 
 
 @dataclass
 class DeployCommandContext:
     stack: str
     cluster_context: ClusterContext
-    deployer: Optional[Deployer]
+    deployer: Deployer | None
 
 
 @dataclass

stack_orchestrator/deploy/deploy_util.py

@@ -13,15 +13,16 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from typing import List, Any
+from typing import Any
+
 from stack_orchestrator.deploy.deploy_types import DeployCommandContext, VolumeMapping
+from stack_orchestrator.opts import opts
 from stack_orchestrator.util import (
     get_parsed_stack_config,
-    get_yaml,
     get_pod_list,
+    get_yaml,
     resolve_compose_file,
 )
-from stack_orchestrator.opts import opts
 
 
 def _container_image_from_service(stack: str, service: str):
@@ -32,7 +33,7 @@ def _container_image_from_service(stack: str, service: str):
     yaml = get_yaml()
     for pod in pods:
         pod_file_path = resolve_compose_file(stack, pod)
-        parsed_pod_file = yaml.load(open(pod_file_path, "r"))
+        parsed_pod_file = yaml.load(open(pod_file_path))
         if "services" in parsed_pod_file:
             services = parsed_pod_file["services"]
             if service in services:
@@ -45,7 +46,7 @@ def _container_image_from_service(stack: str, service: str):
 def parsed_pod_files_map_from_file_names(pod_files):
     parsed_pod_yaml_map: Any = {}
     for pod_file in pod_files:
-        with open(pod_file, "r") as pod_file_descriptor:
+        with open(pod_file) as pod_file_descriptor:
             parsed_pod_file = get_yaml().load(pod_file_descriptor)
             parsed_pod_yaml_map[pod_file] = parsed_pod_file
     if opts.o.debug:
@@ -53,7 +54,7 @@ def parsed_pod_files_map_from_file_names(pod_files):
     return parsed_pod_yaml_map
 
 
-def images_for_deployment(pod_files: List[str]):
+def images_for_deployment(pod_files: list[str]):
     image_set = set()
     parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files)
     # Find the set of images in the pods
@@ -69,7 +70,7 @@ def images_for_deployment(pod_files: List[str]):
     return image_set
 
 
-def _volumes_to_docker(mounts: List[VolumeMapping]):
+def _volumes_to_docker(mounts: list[VolumeMapping]):
     # Example from doc: [("/", "/host"), ("/etc/hosts", "/etc/hosts", "rw")]
     result = []
     for mount in mounts:
@@ -79,7 +80,7 @@ def _volumes_to_docker(mounts: List[VolumeMapping]):
 
 
 def run_container_command(
-    ctx: DeployCommandContext, service: str, command: str, mounts: List[VolumeMapping]
+    ctx: DeployCommandContext, service: str, command: str, mounts: list[VolumeMapping]
 ):
     deployer = ctx.deployer
     if deployer is None:

stack_orchestrator/deploy/deployer.py

@@ -15,7 +15,6 @@
 
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Optional
 
 
 class Deployer(ABC):
@@ -28,7 +27,7 @@ class Deployer(ABC):
         pass
 
     @abstractmethod
-    def update(self):
+    def update_envs(self):
         pass
 
     @abstractmethod
@@ -59,16 +58,23 @@ class Deployer(ABC):
         user=None,
         volumes=None,
         entrypoint=None,
-        env={},
-        ports=[],
+        env=None,
+        ports=None,
         detach=False,
     ):
         pass
 
     @abstractmethod
-    def run_job(self, job_name: str, release_name: Optional[str] = None):
+    def run_job(self, job_name: str, release_name: str | None = None):
         pass
 
+    def prepare(self, skip_cluster_management):
+        """Create cluster infrastructure (namespace, PVs, services) without starting pods.
+
+        Only supported for k8s deployers. Compose deployers raise an error.
+        """
+        raise DeployerException("prepare is only supported for k8s deployments")
+
 
 class DeployerException(Exception):
     def __init__(self, *args: object) -> None:

stack_orchestrator/deploy/deployer_factory.py

@@ -14,14 +14,14 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from stack_orchestrator import constants
-from stack_orchestrator.deploy.k8s.deploy_k8s import (
-    K8sDeployer,
-    K8sDeployerConfigGenerator,
-)
 from stack_orchestrator.deploy.compose.deploy_docker import (
     DockerDeployer,
     DockerDeployerConfigGenerator,
 )
+from stack_orchestrator.deploy.k8s.deploy_k8s import (
+    K8sDeployer,
+    K8sDeployerConfigGenerator,
+)
 
 
 def getDeployerConfigGenerator(type: str, deployment_context):
@@ -34,12 +34,7 @@ def getDeployerConfigGenerator(type: str, deployment_context):
 
 
 def getDeployer(
-    type: str,
-    deployment_context,
-    compose_files,
-    compose_project_name,
-    compose_env_file,
-    job_compose_files=None,
+    type: str, deployment_context, compose_files, compose_project_name, compose_env_file
 ):
     if type == "compose" or type is None:
         return DockerDeployer(
@@ -49,17 +44,13 @@ def getDeployer(
             compose_project_name,
             compose_env_file,
         )
-    elif (
-        type == type == constants.k8s_deploy_type
-        or type == constants.k8s_kind_deploy_type
-    ):
+    elif type == type == constants.k8s_deploy_type or type == constants.k8s_kind_deploy_type:
         return K8sDeployer(
             type,
             deployment_context,
             compose_files,
             compose_project_name,
             compose_env_file,
-            job_compose_files=job_compose_files,
         )
     else:
         print(f"ERROR: deploy-to {type} is not valid")

stack_orchestrator/deploy/deployment.py

@@ -13,28 +13,28 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-import click
-from pathlib import Path
 import subprocess
 import sys
-import time
+from pathlib import Path
+
+import click
+
 from stack_orchestrator import constants
-from stack_orchestrator.deploy.images import push_images_operation
 from stack_orchestrator.deploy.deploy import (
-    up_operation,
+    create_deploy_context,
     down_operation,
-    ps_operation,
-    port_operation,
-    status_operation,
-)
-from stack_orchestrator.deploy.deploy import (
     exec_operation,
     logs_operation,
-    create_deploy_context,
-    update_operation,
+    port_operation,
+    prepare_operation,
+    ps_operation,
+    status_operation,
+    up_operation,
+    update_envs_operation,
 )
 from stack_orchestrator.deploy.deploy_types import DeployCommandContext
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
+from stack_orchestrator.deploy.images import push_images_operation
 
 
 @click.group()
@@ -114,7 +114,7 @@ def up(ctx, stay_attached, skip_cluster_management, extra_args):
 )
 @click.option(
     "--skip-cluster-management/--perform-cluster-management",
-    default=False,
+    default=True,
     help="Skip cluster initialization/tear-down (only for kind-k8s deployments)",
 )
 @click.argument("extra_args", nargs=-1)  # help: command: up <service1> <service2>
@@ -125,14 +125,33 @@ def start(ctx, stay_attached, skip_cluster_management, extra_args):
     up_operation(ctx, services_list, stay_attached, skip_cluster_management)
 
 
-# TODO: remove legacy up command since it's an alias for stop
 @command.command()
-@click.option(
-    "--delete-volumes/--preserve-volumes", default=False, help="delete data volumes"
-)
 @click.option(
     "--skip-cluster-management/--perform-cluster-management",
     default=False,
+    help="Skip cluster initialization (only for kind-k8s deployments)",
+)
+@click.pass_context
+def prepare(ctx, skip_cluster_management):
+    """Create cluster infrastructure without starting pods.
+
+    Sets up the kind cluster, namespace, PVs, PVCs, ConfigMaps, Services,
+    and Ingresses — everything that 'start' does EXCEPT creating the
+    Deployment resource. No pods will be scheduled.
+
+    Use 'start --skip-cluster-management' afterward to create the Deployment
+    and start pods when ready.
+    """
+    ctx.obj = make_deploy_context(ctx)
+    prepare_operation(ctx, skip_cluster_management)
+
+
+# TODO: remove legacy up command since it's an alias for stop
+@command.command()
+@click.option("--delete-volumes/--preserve-volumes", default=False, help="delete data volumes")
+@click.option(
+    "--skip-cluster-management/--perform-cluster-management",
+    default=True,
     help="Skip cluster initialization/tear-down (only for kind-k8s deployments)",
 )
 @click.argument("extra_args", nargs=-1)  # help: command: down <service1> <service2>
@@ -146,12 +165,10 @@ def down(ctx, delete_volumes, skip_cluster_management, extra_args):
 
 # stop is the preferred alias for down
 @command.command()
-@click.option(
-    "--delete-volumes/--preserve-volumes", default=False, help="delete data volumes"
-)
+@click.option("--delete-volumes/--preserve-volumes", default=False, help="delete data volumes")
 @click.option(
     "--skip-cluster-management/--perform-cluster-management",
-    default=False,
+    default=True,
     help="Skip cluster initialization/tear-down (only for kind-k8s deployments)",
 )
 @click.argument("extra_args", nargs=-1)  # help: command: down <service1> <service2>
@@ -210,11 +227,11 @@ def status(ctx):
     status_operation(ctx)
 
 
-@command.command()
+@command.command(name="update-envs")
 @click.pass_context
-def update(ctx):
+def update_envs(ctx):
     ctx.obj = make_deploy_context(ctx)
-    update_operation(ctx)
+    update_envs_operation(ctx)
 
 
 @command.command()
@@ -234,9 +251,7 @@ def run_job(ctx, job_name, helm_release):
 
 @command.command()
 @click.option("--stack-path", help="Path to stack git repo (overrides stored path)")
-@click.option(
-    "--spec-file", help="Path to GitOps spec.yml in repo (e.g., deployment/spec.yml)"
-)
+@click.option("--spec-file", help="Path to GitOps spec.yml in repo (e.g., deployment/spec.yml)")
 @click.option("--config-file", help="Config file to pass to deploy init")
 @click.option(
     "--force",
@@ -270,33 +285,27 @@ def restart(ctx, stack_path, spec_file, config_file, force, expected_ip):
     commands.py on each restart. Use 'deploy init' only for initial
     spec generation, then customize and commit to your operator repo.
     """
-    from stack_orchestrator.util import get_yaml, get_parsed_deployment_spec
     from stack_orchestrator.deploy.deployment_create import create_operation
     from stack_orchestrator.deploy.dns_probe import verify_dns_via_probe
+    from stack_orchestrator.util import get_parsed_deployment_spec, get_yaml
 
     deployment_context: DeploymentContext = ctx.obj
 
     # Get current spec info (before git pull)
     current_spec = deployment_context.spec
     current_http_proxy = current_spec.get_http_proxy()
-    current_hostname = (
-        current_http_proxy[0]["host-name"] if current_http_proxy else None
-    )
+    current_hostname = current_http_proxy[0]["host-name"] if current_http_proxy else None
 
     # Resolve stack source path
     if stack_path:
         stack_source = Path(stack_path).resolve()
     else:
         # Try to get from deployment.yml
-        deployment_file = (
-            deployment_context.deployment_dir / constants.deployment_file_name
-        )
+        deployment_file = deployment_context.deployment_dir / constants.deployment_file_name
         deployment_data = get_yaml().load(open(deployment_file))
         stack_source_str = deployment_data.get("stack-source")
         if not stack_source_str:
-            print(
-                "Error: No stack-source in deployment.yml and --stack-path not provided"
-            )
+            print("Error: No stack-source in deployment.yml and --stack-path not provided")
             print("Use --stack-path to specify the stack git repository location")
             sys.exit(1)
         stack_source = Path(stack_source_str)
@@ -312,9 +321,7 @@ def restart(ctx, stack_path, spec_file, config_file, force, expected_ip):
 
     # Step 1: Git pull (brings in updated spec.yml from operator's repo)
     print("\n[1/4] Pulling latest code from stack repository...")
-    git_result = subprocess.run(
-        ["git", "pull"], cwd=stack_source, capture_output=True, text=True
-    )
+    git_result = subprocess.run(["git", "pull"], cwd=stack_source, capture_output=True, text=True)
     if git_result.returncode != 0:
         print(f"Git pull failed: {git_result.stderr}")
         sys.exit(1)
@@ -386,17 +393,13 @@ def restart(ctx, stack_path, spec_file, config_file, force, expected_ip):
     # Stop deployment
     print("\n[4/4] Restarting deployment...")
     ctx.obj = make_deploy_context(ctx)
-    down_operation(
-        ctx, delete_volumes=False, extra_args_list=[], skip_cluster_management=True
-    )
+    down_operation(ctx, delete_volumes=False, extra_args_list=[], skip_cluster_management=True)
 
-    # Brief pause to ensure clean shutdown
-    time.sleep(5)
+    # Namespace deletion wait is handled by _ensure_namespace() in
+    # the deployer — no fixed sleep needed here.
 
     # Start deployment
-    up_operation(
-        ctx, services_list=None, stay_attached=False, skip_cluster_management=True
-    )
+    up_operation(ctx, services_list=None, stay_attached=False, skip_cluster_management=True)
 
     print("\n=== Restart Complete ===")
     print("Deployment restarted with git-tracked configuration.")

stack_orchestrator/deploy/deployment_context.py

@@ -18,9 +18,9 @@ import os
 from pathlib import Path
 
 from stack_orchestrator import constants
-from stack_orchestrator.util import get_yaml
-from stack_orchestrator.deploy.stack import Stack
 from stack_orchestrator.deploy.spec import Spec
+from stack_orchestrator.deploy.stack import Stack
+from stack_orchestrator.util import get_yaml
 
 
 class DeploymentContext:
@@ -58,7 +58,7 @@ class DeploymentContext:
         self.stack.init_from_file(self.get_stack_file())
         deployment_file_path = self.get_deployment_file()
         if deployment_file_path.exists():
-            obj = get_yaml().load(open(deployment_file_path, "r"))
+            obj = get_yaml().load(open(deployment_file_path))
             self.id = obj[constants.cluster_id_key]
         # Handle the case of a legacy deployment with no file
         # Code below is intended to match the output from _make_default_cluster_name()
@@ -75,7 +75,7 @@ class DeploymentContext:
             raise ValueError(f"File is not inside deployment directory: {file_path}")
 
         yaml = get_yaml()
-        with open(file_path, "r") as f:
+        with open(file_path) as f:
             yaml_data = yaml.load(f)
 
         modifier_func(yaml_data)

stack_orchestrator/deploy/deployment_create.py

@@ -13,44 +13,44 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-import click
-from importlib import util
+import base64
+import filecmp
 import json
 import os
-import re
-import base64
-from pathlib import Path
-from typing import List, Optional
 import random
-from shutil import copy, copyfile, copytree, rmtree
-from secrets import token_hex
+import re
 import sys
-import filecmp
 import tempfile
+from importlib import util
+from pathlib import Path
+from secrets import token_hex
+from shutil import copy, copyfile, copytree, rmtree
+
+import click
 
 from stack_orchestrator import constants
-from stack_orchestrator.opts import opts
-from stack_orchestrator.util import (
-    get_stack_path,
-    get_parsed_deployment_spec,
-    get_parsed_stack_config,
-    global_options,
-    get_yaml,
-    get_pod_list,
-    get_pod_file_path,
-    pod_has_scripts,
-    get_pod_script_paths,
-    get_plugin_code_paths,
-    error_exit,
-    env_var_map_from_file,
-    resolve_config_dir,
-    get_job_list,
-    get_job_file_path,
-)
-from stack_orchestrator.deploy.spec import Spec
 from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand
 from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
+from stack_orchestrator.deploy.spec import Spec
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import (
+    env_var_map_from_file,
+    error_exit,
+    get_job_file_path,
+    get_job_list,
+    get_parsed_deployment_spec,
+    get_parsed_stack_config,
+    get_plugin_code_paths,
+    get_pod_file_path,
+    get_pod_list,
+    get_pod_script_paths,
+    get_stack_path,
+    get_yaml,
+    global_options,
+    pod_has_scripts,
+    resolve_config_dir,
+)
 
 
 def _make_default_deployment_dir():
@@ -66,7 +66,7 @@ def _get_ports(stack):
         pod_file_path = get_pod_file_path(stack, parsed_stack, pod)
         if pod_file_path is None:
             continue
-        parsed_pod_file = yaml.load(open(pod_file_path, "r"))
+        parsed_pod_file = yaml.load(open(pod_file_path))
         if "services" in parsed_pod_file:
             for svc_name, svc in parsed_pod_file["services"].items():
                 if "ports" in svc:
@@ -102,7 +102,7 @@ def _get_named_volumes(stack):
         pod_file_path = get_pod_file_path(stack, parsed_stack, pod)
         if pod_file_path is None:
             continue
-        parsed_pod_file = yaml.load(open(pod_file_path, "r"))
+        parsed_pod_file = yaml.load(open(pod_file_path))
         if "volumes" in parsed_pod_file:
             volumes = parsed_pod_file["volumes"]
             for volume in volumes.keys():
@@ -132,9 +132,7 @@ def _create_bind_dir_if_relative(volume, path_string, compose_dir):
         absolute_path.mkdir(parents=True, exist_ok=True)
     else:
         if not path.exists():
-            print(
-                f"WARNING: mount path for volume {volume} does not exist: {path_string}"
-            )
+            print(f"WARNING: mount path for volume {volume} does not exist: {path_string}")
 
 
 # See:
@@ -151,9 +149,7 @@ def _fixup_pod_file(pod, spec, compose_dir):
                     volume_spec = spec_volumes[volume]
                     if volume_spec:
                         volume_spec_fixedup = (
-                            volume_spec
-                            if Path(volume_spec).is_absolute()
-                            else f".{volume_spec}"
+                            volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}"
                         )
                         _create_bind_dir_if_relative(volume, volume_spec, compose_dir)
                         # this is Docker specific
@@ -265,25 +261,6 @@ def call_stack_deploy_create(deployment_context, extra_args):
                 imported_stack.create(deployment_context, extra_args)
 
 
-def call_stack_deploy_start(deployment_context):
-    """Call start() hooks after k8s deployments and jobs are created.
-
-    The start() hook receives the DeploymentContext, allowing stacks to
-    create additional k8s resources (Services, etc.) in the deployment namespace.
-    The namespace can be derived as f"laconic-{deployment_context.id}".
-    """
-    python_file_paths = _commands_plugin_paths(deployment_context.stack.name)
-    for python_file_path in python_file_paths:
-        if python_file_path.exists():
-            spec = util.spec_from_file_location("commands", python_file_path)
-            if spec is None or spec.loader is None:
-                continue
-            imported_stack = util.module_from_spec(spec)
-            spec.loader.exec_module(imported_stack)
-            if _has_method(imported_stack, "start"):
-                imported_stack.start(deployment_context)
-
-
 # Inspect the pod yaml to find config files referenced in subdirectories
 # other than the one associated with the pod
 def _find_extra_config_dirs(parsed_pod_file, pod):
@@ -347,10 +324,7 @@ def _get_mapped_ports(stack: str, map_recipe: str):
                         else:
                             print("Error: bad map_recipe")
             else:
-                print(
-                    f"Error: --map-ports-to-host must specify one of: "
-                    f"{port_map_recipes}"
-                )
+                print(f"Error: --map-ports-to-host must specify one of: " f"{port_map_recipes}")
                 sys.exit(1)
     return ports
 
@@ -375,9 +349,7 @@ def _parse_config_variables(variable_values: str):
 
 @click.command()
 @click.option("--config", help="Provide config variables for the deployment")
-@click.option(
-    "--config-file", help="Provide config variables in a file for the deployment"
-)
+@click.option("--config-file", help="Provide config variables in a file for the deployment")
 @click.option("--kube-config", help="Provide a config file for a k8s deployment")
 @click.option(
     "--image-registry",
@@ -391,9 +363,7 @@ def _parse_config_variables(variable_values: str):
     "localhost-same, any-same, localhost-fixed-random, any-fixed-random",
 )
 @click.pass_context
-def init(
-    ctx, config, config_file, kube_config, image_registry, output, map_ports_to_host
-):
+def init(ctx, config, config_file, kube_config, image_registry, output, map_ports_to_host):
     stack = global_options(ctx).stack
     deployer_type = ctx.obj.deployer.type
     deploy_command_context = ctx.obj
@@ -440,13 +410,9 @@ def init_operation(
     else:
         # Check for --kube-config supplied for non-relevant deployer types
         if kube_config is not None:
-            error_exit(
-                f"--kube-config is not allowed with a {deployer_type} deployment"
-            )
+            error_exit(f"--kube-config is not allowed with a {deployer_type} deployment")
         if image_registry is not None:
-            error_exit(
-                f"--image-registry is not allowed with a {deployer_type} deployment"
-            )
+            error_exit(f"--image-registry is not allowed with a {deployer_type} deployment")
     if default_spec_file_content:
         spec_file_content.update(default_spec_file_content)
     config_variables = _parse_config_variables(config)
@@ -496,14 +462,9 @@ def init_operation(
             spec_file_content["volumes"] = {**volume_descriptors, **orig_volumes}
         if configmap_descriptors:
             spec_file_content["configmaps"] = configmap_descriptors
-        if "k8s" in deployer_type:
-            if "secrets" not in spec_file_content:
-                spec_file_content["secrets"] = {}
 
     if opts.o.debug:
-        print(
-            f"Creating spec file for stack: {stack} with content: {spec_file_content}"
-        )
+        print(f"Creating spec file for stack: {stack} with content: {spec_file_content}")
 
     with open(output, "w") as output_file:
         get_yaml().dump(spec_file_content, output_file)
@@ -519,7 +480,8 @@ def _generate_and_store_secrets(config_vars: dict, deployment_name: str):
     Called by `deploy create` - generates fresh secrets and stores them.
     Returns the generated secrets dict for reference.
     """
-    from kubernetes import client, config as k8s_config
+    from kubernetes import client
+    from kubernetes import config as k8s_config
 
     secrets = {}
     for name, value in config_vars.items():
@@ -548,9 +510,7 @@ def _generate_and_store_secrets(config_vars: dict, deployment_name: str):
         try:
             k8s_config.load_incluster_config()
         except Exception:
-            print(
-                "Warning: Could not load kube config, secrets will not be stored in K8s"
-            )
+            print("Warning: Could not load kube config, secrets will not be stored in K8s")
             return secrets
 
     v1 = client.CoreV1Api()
@@ -577,7 +537,7 @@ def _generate_and_store_secrets(config_vars: dict, deployment_name: str):
     return secrets
 
 
-def create_registry_secret(spec: Spec, deployment_name: str) -> Optional[str]:
+def create_registry_secret(spec: Spec, deployment_name: str) -> str | None:
     """Create K8s docker-registry secret from spec + environment.
 
     Reads registry configuration from spec.yml and creates a Kubernetes
@@ -590,7 +550,8 @@ def create_registry_secret(spec: Spec, deployment_name: str) -> Optional[str]:
     Returns:
         The secret name if created, None if no registry config
     """
-    from kubernetes import client, config as k8s_config
+    from kubernetes import client
+    from kubernetes import config as k8s_config
 
     registry_config = spec.get_image_registry_config()
     if not registry_config:
@@ -607,17 +568,12 @@ def create_registry_secret(spec: Spec, deployment_name: str) -> Optional[str]:
     assert token_env is not None
     token = os.environ.get(token_env)
     if not token:
-        print(
-            f"Warning: Registry token env var '{token_env}' not set, "
-            "skipping registry secret"
-        )
+        print(f"Warning: Registry token env var '{token_env}' not set, " "skipping registry secret")
         return None
 
     # Create dockerconfigjson format (Docker API uses "password" field for tokens)
     auth = base64.b64encode(f"{username}:{token}".encode()).decode()
-    docker_config = {
-        "auths": {server: {"username": username, "password": token, "auth": auth}}
-    }
+    docker_config = {"auths": {server: {"username": username, "password": token, "auth": auth}}}
 
     # Secret name derived from deployment name
     secret_name = f"{deployment_name}-registry"
@@ -637,11 +593,7 @@ def create_registry_secret(spec: Spec, deployment_name: str) -> Optional[str]:
 
     k8s_secret = client.V1Secret(
         metadata=client.V1ObjectMeta(name=secret_name),
-        data={
-            ".dockerconfigjson": base64.b64encode(
-                json.dumps(docker_config).encode()
-            ).decode()
-        },
+        data={".dockerconfigjson": base64.b64encode(json.dumps(docker_config).encode()).decode()},
         type="kubernetes.io/dockerconfigjson",
     )
 
@@ -658,17 +610,14 @@ def create_registry_secret(spec: Spec, deployment_name: str) -> Optional[str]:
     return secret_name
 
 
-def _write_config_file(
-    spec_file: Path, config_env_file: Path, deployment_name: Optional[str] = None
-):
+def _write_config_file(spec_file: Path, config_env_file: Path, deployment_name: str | None = None):
     spec_content = get_parsed_deployment_spec(spec_file)
     config_vars = spec_content.get("config", {}) or {}
 
     # Generate and store secrets in K8s if deployment_name provided and tokens exist
     if deployment_name and config_vars:
         has_generate_tokens = any(
-            isinstance(v, str) and GENERATE_TOKEN_PATTERN.search(v)
-            for v in config_vars.values()
+            isinstance(v, str) and GENERATE_TOKEN_PATTERN.search(v) for v in config_vars.values()
         )
         if has_generate_tokens:
             _generate_and_store_secrets(config_vars, deployment_name)
@@ -691,13 +640,13 @@ def _write_kube_config_file(external_path: Path, internal_path: Path):
     copyfile(external_path, internal_path)
 
 
-def _copy_files_to_directory(file_paths: List[Path], directory: Path):
+def _copy_files_to_directory(file_paths: list[Path], directory: Path):
     for path in file_paths:
         # Using copy to preserve the execute bit
         copy(path, os.path.join(directory, os.path.basename(path)))
 
 
-def _create_deployment_file(deployment_dir: Path, stack_source: Optional[Path] = None):
+def _create_deployment_file(deployment_dir: Path, stack_source: Path | None = None):
     deployment_file_path = deployment_dir.joinpath(constants.deployment_file_name)
     cluster = f"{constants.cluster_name_prefix}{token_hex(8)}"
     deployment_content = {constants.cluster_id_key: cluster}
@@ -723,9 +672,7 @@ def _check_volume_definitions(spec):
 
 
 @click.command()
-@click.option(
-    "--spec-file", required=True, help="Spec file to use to create this deployment"
-)
+@click.option("--spec-file", required=True, help="Spec file to use to create this deployment")
 @click.option("--deployment-dir", help="Create deployment files in this directory")
 @click.option(
     "--update",
@@ -779,9 +726,7 @@ def create_operation(
     initial_peers=None,
     extra_args=(),
 ):
-    parsed_spec = Spec(
-        os.path.abspath(spec_file), get_parsed_deployment_spec(spec_file)
-    )
+    parsed_spec = Spec(os.path.abspath(spec_file), get_parsed_deployment_spec(spec_file))
     _check_volume_definitions(parsed_spec)
     stack_name = parsed_spec["stack"]
     deployment_type = parsed_spec[constants.deploy_to_key]
@@ -838,9 +783,7 @@ def create_operation(
             # Exclude config file to preserve deployment settings
             # (XXX breaks passing config vars from spec)
             exclude_patterns = ["data", "data/*", constants.config_file_name]
-            _safe_copy_tree(
-                temp_dir, deployment_dir_path, exclude_patterns=exclude_patterns
-            )
+            _safe_copy_tree(temp_dir, deployment_dir_path, exclude_patterns=exclude_patterns)
         finally:
             # Clean up temp dir
             rmtree(temp_dir)
@@ -863,18 +806,14 @@ def create_operation(
     deployment_context = DeploymentContext()
     deployment_context.init(deployment_dir_path)
     # Call the deployer to generate any deployer-specific files (e.g. for kind)
-    deployer_config_generator = getDeployerConfigGenerator(
-        deployment_type, deployment_context
-    )
+    deployer_config_generator = getDeployerConfigGenerator(deployment_type, deployment_context)
     # TODO: make deployment_dir_path a Path above
     if deployer_config_generator is not None:
         deployer_config_generator.generate(deployment_dir_path)
-    call_stack_deploy_create(
-        deployment_context, [network_dir, initial_peers, *extra_args]
-    )
+    call_stack_deploy_create(deployment_context, [network_dir, initial_peers, *extra_args])
 
 
-def _safe_copy_tree(src: Path, dst: Path, exclude_patterns: Optional[List[str]] = None):
+def _safe_copy_tree(src: Path, dst: Path, exclude_patterns: list[str] | None = None):
     """
     Recursively copy a directory tree, backing up changed files with .bak suffix.
 
@@ -895,11 +834,7 @@ def _safe_copy_tree(src: Path, dst: Path, exclude_patterns: Optional[List[str]]
 
     def safe_copy_file(src_file: Path, dst_file: Path):
         """Copy file, backing up destination if it differs."""
-        if (
-            dst_file.exists()
-            and not dst_file.is_dir()
-            and not filecmp.cmp(src_file, dst_file)
-        ):
+        if dst_file.exists() and not dst_file.is_dir() and not filecmp.cmp(src_file, dst_file):
             os.rename(dst_file, f"{dst_file}.bak")
         copy(src_file, dst_file)
 
@@ -925,7 +860,7 @@ def _write_deployment_files(
     stack_name: str,
     deployment_type: str,
     include_deployment_file: bool = True,
-    stack_source: Optional[Path] = None,
+    stack_source: Path | None = None,
 ):
     """
     Write deployment files to target directory.
@@ -953,9 +888,7 @@ def _write_deployment_files(
     # Use stack_name as deployment_name for K8s secret naming
     # Extract just the name part if stack_name is a path ("path/to/stack" -> "stack")
     deployment_name = Path(stack_name).name.replace("_", "-")
-    _write_config_file(
-        spec_file, target_dir.joinpath(constants.config_file_name), deployment_name
-    )
+    _write_config_file(spec_file, target_dir.joinpath(constants.config_file_name), deployment_name)
 
     # Copy any k8s config file into the target dir
     if deployment_type == "k8s":
@@ -976,7 +909,7 @@ def _write_deployment_files(
         pod_file_path = get_pod_file_path(stack_name, parsed_stack, pod)
         if pod_file_path is None:
             continue
-        parsed_pod_file = yaml.load(open(pod_file_path, "r"))
+        parsed_pod_file = yaml.load(open(pod_file_path))
         extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod)
         destination_pod_dir = destination_pods_dir.joinpath(pod)
         os.makedirs(destination_pod_dir, exist_ok=True)
@@ -984,7 +917,7 @@ def _write_deployment_files(
             print(f"extra config dirs: {extra_config_dirs}")
         _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir)
         with open(
-            destination_compose_dir.joinpath("docker-compose-%s.yml" % pod), "w"
+            destination_compose_dir.joinpath(f"docker-compose-{pod}.yml"), "w"
         ) as output_file:
             yaml.dump(parsed_pod_file, output_file)
 
@@ -1004,7 +937,13 @@ def _write_deployment_files(
             script_paths = get_pod_script_paths(parsed_stack, pod)
             _copy_files_to_directory(script_paths, destination_script_dir)
 
-        if not parsed_spec.is_kubernetes_deployment():
+        if parsed_spec.is_kubernetes_deployment():
+            for configmap in parsed_spec.get_configmaps():
+                source_config_dir = resolve_config_dir(stack_name, configmap)
+                if os.path.exists(source_config_dir):
+                    destination_config_dir = target_dir.joinpath("configmaps", configmap)
+                    copytree(source_config_dir, destination_config_dir, dirs_exist_ok=True)
+        else:
             # TODO:
             # This is odd - looks up config dir that matches a volume name,
             # then copies as a mount dir?
@@ -1026,33 +965,18 @@ def _write_deployment_files(
                             dirs_exist_ok=True,
                         )
 
-    # Copy configmap directories for k8s deployments (outside the pod loop
-    # so this works for jobs-only stacks too)
-    if parsed_spec.is_kubernetes_deployment():
-        for configmap in parsed_spec.get_configmaps():
-            source_config_dir = resolve_config_dir(stack_name, configmap)
-            if os.path.exists(source_config_dir):
-                destination_config_dir = target_dir.joinpath(
-                    "configmaps", configmap
-                )
-                copytree(
-                    source_config_dir, destination_config_dir, dirs_exist_ok=True
-                )
-
-    # Copy the job files into the target dir
+    # Copy the job files into the target dir (for Docker deployments)
     jobs = get_job_list(parsed_stack)
-    if jobs:
+    if jobs and not parsed_spec.is_kubernetes_deployment():
         destination_compose_jobs_dir = target_dir.joinpath("compose-jobs")
         os.makedirs(destination_compose_jobs_dir, exist_ok=True)
         for job in jobs:
             job_file_path = get_job_file_path(stack_name, parsed_stack, job)
             if job_file_path and job_file_path.exists():
-                parsed_job_file = yaml.load(open(job_file_path, "r"))
+                parsed_job_file = yaml.load(open(job_file_path))
                 _fixup_pod_file(parsed_job_file, parsed_spec, destination_compose_dir)
                 with open(
-                    destination_compose_jobs_dir.joinpath(
-                        "docker-compose-%s.yml" % job
-                    ),
+                    destination_compose_jobs_dir.joinpath(f"docker-compose-{job}.yml"),
                     "w",
                 ) as output_file:
                     yaml.dump(parsed_job_file, output_file)
@@ -1067,18 +991,14 @@ def _write_deployment_files(
 @click.option("--node-moniker", help="Moniker for this node")
 @click.option("--chain-id", help="The new chain id")
 @click.option("--key-name", help="Name for new node key")
-@click.option(
-    "--gentx-files", help="List of comma-delimited gentx filenames from other nodes"
-)
+@click.option("--gentx-files", help="List of comma-delimited gentx filenames from other nodes")
 @click.option(
     "--gentx-addresses",
     type=str,
     help="List of comma-delimited validator addresses for other nodes",
 )
 @click.option("--genesis-file", help="Genesis file for the network")
-@click.option(
-    "--initialize-network", is_flag=True, default=False, help="Initialize phase"
-)
+@click.option("--initialize-network", is_flag=True, default=False, help="Initialize phase")
 @click.option("--join-network", is_flag=True, default=False, help="Join phase")
 @click.option("--connect-network", is_flag=True, default=False, help="Connect phase")
 @click.option("--create-network", is_flag=True, default=False, help="Create phase")

stack_orchestrator/deploy/dns_probe.py

@@ -6,7 +6,7 @@
 import secrets
 import socket
 import time
-from typing import Optional
+
 import requests
 from kubernetes import client
 
@@ -15,7 +15,8 @@ def get_server_egress_ip() -> str:
     """Get this server's public egress IP via ipify."""
     response = requests.get("https://api.ipify.org", timeout=10)
     response.raise_for_status()
-    return response.text.strip()
+    result: str = response.text.strip()
+    return result
 
 
 def resolve_hostname(hostname: str) -> list[str]:
@@ -27,7 +28,7 @@ def resolve_hostname(hostname: str) -> list[str]:
         return []
 
 
-def verify_dns_simple(hostname: str, expected_ip: Optional[str] = None) -> bool:
+def verify_dns_simple(hostname: str, expected_ip: str | None = None) -> bool:
     """Simple DNS verification - check hostname resolves to expected IP.
 
     If expected_ip not provided, uses server's egress IP.
@@ -98,9 +99,7 @@ def delete_probe_ingress(namespace: str = "default"):
     """Delete the temporary probe ingress."""
     networking_api = client.NetworkingV1Api()
     try:
-        networking_api.delete_namespaced_ingress(
-            name="laconic-dns-probe", namespace=namespace
-        )
+        networking_api.delete_namespaced_ingress(name="laconic-dns-probe", namespace=namespace)
     except client.exceptions.ApiException:
         pass  # Ignore if already deleted
 

stack_orchestrator/deploy/images.py

@@ -13,15 +13,14 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from typing import Set
 
 from python_on_whales import DockerClient
 
 from stack_orchestrator import constants
-from stack_orchestrator.opts import opts
-from stack_orchestrator.deploy.deployment_context import DeploymentContext
 from stack_orchestrator.deploy.deploy_types import DeployCommandContext
 from stack_orchestrator.deploy.deploy_util import images_for_deployment
+from stack_orchestrator.deploy.deployment_context import DeploymentContext
+from stack_orchestrator.opts import opts
 
 
 def _image_needs_pushed(image: str):
@@ -32,9 +31,7 @@ def _image_needs_pushed(image: str):
 def _remote_tag_for_image(image: str, remote_repo_url: str):
     # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
     major_parts = image.split("/", 2)
-    image_name_with_version = (
-        major_parts[1] if 2 == len(major_parts) else major_parts[0]
-    )
+    image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
     (image_name, image_version) = image_name_with_version.split(":")
     if image_version == "local":
         return f"{remote_repo_url}/{image_name}:deploy"
@@ -63,18 +60,14 @@ def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
 
     docker = DockerClient()
     remote_tag = _remote_tag_for_image(local_tag, remote_repo_url)
-    new_remote_tags = [
-        _remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags
-    ]
+    new_remote_tags = [_remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
     docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags)
 
 
 def remote_tag_for_image_unique(image: str, remote_repo_url: str, deployment_id: str):
     # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
     major_parts = image.split("/", 2)
-    image_name_with_version = (
-        major_parts[1] if 2 == len(major_parts) else major_parts[0]
-    )
+    image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
     (image_name, image_version) = image_name_with_version.split(":")
     if image_version == "local":
         # Salt the tag with part of the deployment id to make it unique to this
@@ -91,24 +84,20 @@ def push_images_operation(
 ):
     # Get the list of images for the stack
     cluster_context = command_context.cluster_context
-    images: Set[str] = images_for_deployment(cluster_context.compose_files)
+    images: set[str] = images_for_deployment(cluster_context.compose_files)
     # Tag the images for the remote repo
     remote_repo_url = deployment_context.spec.obj[constants.image_registry_key]
     docker = DockerClient()
     for image in images:
         if _image_needs_pushed(image):
-            remote_tag = remote_tag_for_image_unique(
-                image, remote_repo_url, deployment_context.id
-            )
+            remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id)
             if opts.o.verbose:
                 print(f"Tagging {image} to {remote_tag}")
             docker.image.tag(image, remote_tag)
     # Run docker push commands to upload
     for image in images:
         if _image_needs_pushed(image):
-            remote_tag = remote_tag_for_image_unique(
-                image, remote_repo_url, deployment_context.id
-            )
+            remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id)
             if opts.o.verbose:
                 print(f"Pushing image {remote_tag}")
             docker.image.push(remote_tag)

stack_orchestrator/deploy/k8s/cluster_info.py

@@ -13,33 +13,31 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-import os
 import base64
+import os
+from typing import Any
 
 from kubernetes import client
-from typing import Any, List, Optional, Set
 
-from stack_orchestrator.opts import opts
-from stack_orchestrator.util import env_var_map_from_file
+from stack_orchestrator.deploy.deploy_types import DeployEnvVars
+from stack_orchestrator.deploy.deploy_util import (
+    images_for_deployment,
+    parsed_pod_files_map_from_file_names,
+)
+from stack_orchestrator.deploy.images import remote_tag_for_image_unique
 from stack_orchestrator.deploy.k8s.helpers import (
+    envs_from_compose_file,
+    envs_from_environment_variables_map,
+    get_kind_pv_bind_mount_path,
+    merge_envs,
     named_volumes_from_pod_files,
+    translate_sidecar_service_names,
     volume_mounts_for_service,
     volumes_for_pod_files,
 )
-from stack_orchestrator.deploy.k8s.helpers import get_kind_pv_bind_mount_path
-from stack_orchestrator.deploy.k8s.helpers import (
-    envs_from_environment_variables_map,
-    envs_from_compose_file,
-    merge_envs,
-    translate_sidecar_service_names,
-)
-from stack_orchestrator.deploy.deploy_util import (
-    parsed_pod_files_map_from_file_names,
-    images_for_deployment,
-)
-from stack_orchestrator.deploy.deploy_types import DeployEnvVars
-from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits
-from stack_orchestrator.deploy.images import remote_tag_for_image_unique
+from stack_orchestrator.deploy.spec import ResourceLimits, Resources, Spec
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import env_var_map_from_file
 
 DEFAULT_VOLUME_RESOURCES = Resources({"reservations": {"storage": "2Gi"}})
 
@@ -52,7 +50,7 @@ DEFAULT_CONTAINER_RESOURCES = Resources(
 
 
 def to_k8s_resource_requirements(resources: Resources) -> client.V1ResourceRequirements:
-    def to_dict(limits: Optional[ResourceLimits]):
+    def to_dict(limits: ResourceLimits | None):
         if not limits:
             return None
 
@@ -72,43 +70,26 @@ def to_k8s_resource_requirements(resources: Resources) -> client.V1ResourceRequi
 
 class ClusterInfo:
     parsed_pod_yaml_map: Any
-    parsed_job_yaml_map: Any
-    image_set: Set[str] = set()
+    image_set: set[str] = set()
     app_name: str
-    stack_name: str
     environment_variables: DeployEnvVars
     spec: Spec
 
     def __init__(self) -> None:
-        self.parsed_job_yaml_map = {}
+        pass
 
-    def int(self, pod_files: List[str], compose_env_file, deployment_name, spec: Spec, stack_name=""):
+    def int(self, pod_files: list[str], compose_env_file, deployment_name, spec: Spec):
         self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files)
         # Find the set of images in the pods
         self.image_set = images_for_deployment(pod_files)
         # Filter out None values from env file
-        env_vars = {
-            k: v for k, v in env_var_map_from_file(compose_env_file).items() if v
-        }
+        env_vars = {k: v for k, v in env_var_map_from_file(compose_env_file).items() if v}
         self.environment_variables = DeployEnvVars(env_vars)
         self.app_name = deployment_name
-        self.stack_name = stack_name
         self.spec = spec
         if opts.o.debug:
             print(f"Env vars: {self.environment_variables.map}")
 
-    def init_jobs(self, job_files: List[str]):
-        """Initialize parsed job YAML map from job compose files."""
-        self.parsed_job_yaml_map = parsed_pod_files_map_from_file_names(job_files)
-        if opts.o.debug:
-            print(f"Parsed job yaml map: {self.parsed_job_yaml_map}")
-
-    def _all_named_volumes(self) -> list:
-        """Return named volumes from both pod and job compose files."""
-        volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
-        volumes.extend(named_volumes_from_pod_files(self.parsed_job_yaml_map))
-        return volumes
-
     def get_nodeports(self):
         nodeports = []
         for pod_name in self.parsed_pod_yaml_map:
@@ -139,8 +120,7 @@ class ClusterInfo:
                         service = client.V1Service(
                             metadata=client.V1ObjectMeta(
                                 name=(
-                                    f"{self.app_name}-nodeport-"
-                                    f"{pod_port}-{protocol.lower()}"
+                                    f"{self.app_name}-nodeport-" f"{pod_port}-{protocol.lower()}"
                                 ),
                                 labels={"app": self.app_name},
                             ),
@@ -160,9 +140,7 @@ class ClusterInfo:
                         nodeports.append(service)
         return nodeports
 
-    def get_ingress(
-        self, use_tls=False, certificate=None, cluster_issuer="letsencrypt-prod"
-    ):
+    def get_ingress(self, use_tls=False, certificate=None, cluster_issuer="letsencrypt-prod"):
         # No ingress for a deployment that has no http-proxy defined, for now
         http_proxy_info_list = self.spec.get_http_proxy()
         ingress = None
@@ -177,9 +155,7 @@ class ClusterInfo:
             tls = (
                 [
                     client.V1IngressTLS(
-                        hosts=certificate["spec"]["dnsNames"]
-                        if certificate
-                        else [host_name],
+                        hosts=certificate["spec"]["dnsNames"] if certificate else [host_name],
                         secret_name=certificate["spec"]["secretName"]
                         if certificate
                         else f"{self.app_name}-tls",
@@ -252,8 +228,7 @@ class ClusterInfo:
             return None
 
         service_ports = [
-            client.V1ServicePort(port=p, target_port=p, name=f"port-{p}")
-            for p in sorted(ports_set)
+            client.V1ServicePort(port=p, target_port=p, name=f"port-{p}") for p in sorted(ports_set)
         ]
 
         service = client.V1Service(
@@ -272,26 +247,20 @@ class ClusterInfo:
     def get_pvcs(self):
         result = []
         spec_volumes = self.spec.get_volumes()
-        named_volumes = self._all_named_volumes()
-        global_resources = self.spec.get_volume_resources()
-        if not global_resources:
-            global_resources = DEFAULT_VOLUME_RESOURCES
+        named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        resources = self.spec.get_volume_resources()
+        if not resources:
+            resources = DEFAULT_VOLUME_RESOURCES
         if opts.o.debug:
             print(f"Spec Volumes: {spec_volumes}")
             print(f"Named Volumes: {named_volumes}")
-            print(f"Resources: {global_resources}")
+            print(f"Resources: {resources}")
         for volume_name, volume_path in spec_volumes.items():
             if volume_name not in named_volumes:
                 if opts.o.debug:
                     print(f"{volume_name} not in pod files")
                 continue
 
-            # Per-volume resources override global, which overrides default.
-            vol_resources = (
-                self.spec.get_volume_resources_for(volume_name)
-                or global_resources
-            )
-
             labels = {
                 "app": self.app_name,
                 "volume-label": f"{self.app_name}-{volume_name}",
@@ -307,13 +276,11 @@ class ClusterInfo:
             spec = client.V1PersistentVolumeClaimSpec(
                 access_modes=["ReadWriteOnce"],
                 storage_class_name=storage_class_name,
-                resources=to_k8s_resource_requirements(vol_resources),
+                resources=to_k8s_resource_requirements(resources),
                 volume_name=k8s_volume_name,
             )
             pvc = client.V1PersistentVolumeClaim(
-                metadata=client.V1ObjectMeta(
-                    name=f"{self.app_name}-{volume_name}", labels=labels
-                ),
+                metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}", labels=labels),
                 spec=spec,
             )
             result.append(pvc)
@@ -322,7 +289,7 @@ class ClusterInfo:
     def get_configmaps(self):
         result = []
         spec_configmaps = self.spec.get_configmaps()
-        named_volumes = self._all_named_volumes()
+        named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
         for cfg_map_name, cfg_map_path in spec_configmaps.items():
             if cfg_map_name not in named_volumes:
                 if opts.o.debug:
@@ -330,9 +297,7 @@ class ClusterInfo:
                 continue
 
             if not cfg_map_path.startswith("/") and self.spec.file_path is not None:
-                cfg_map_path = os.path.join(
-                    os.path.dirname(str(self.spec.file_path)), cfg_map_path
-                )
+                cfg_map_path = os.path.join(os.path.dirname(str(self.spec.file_path)), cfg_map_path)
 
             # Read in all the files at a single-level of the directory.
             # This mimics the behavior of
@@ -341,9 +306,7 @@ class ClusterInfo:
             for f in os.listdir(cfg_map_path):
                 full_path = os.path.join(cfg_map_path, f)
                 if os.path.isfile(full_path):
-                    data[f] = base64.b64encode(open(full_path, "rb").read()).decode(
-                        "ASCII"
-                    )
+                    data[f] = base64.b64encode(open(full_path, "rb").read()).decode("ASCII")
 
             spec = client.V1ConfigMap(
                 metadata=client.V1ObjectMeta(
@@ -358,10 +321,10 @@ class ClusterInfo:
     def get_pvs(self):
         result = []
         spec_volumes = self.spec.get_volumes()
-        named_volumes = self._all_named_volumes()
-        global_resources = self.spec.get_volume_resources()
-        if not global_resources:
-            global_resources = DEFAULT_VOLUME_RESOURCES
+        named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        resources = self.spec.get_volume_resources()
+        if not resources:
+            resources = DEFAULT_VOLUME_RESOURCES
         for volume_name, volume_path in spec_volumes.items():
             # We only need to create a volume if it is fully qualified HostPath.
             # Otherwise, we create the PVC and expect the node to allocate the volume
@@ -390,10 +353,6 @@ class ClusterInfo:
                     )
                     continue
 
-            vol_resources = (
-                self.spec.get_volume_resources_for(volume_name)
-                or global_resources
-            )
             if self.spec.is_kind_deployment():
                 host_path = client.V1HostPathVolumeSource(
                     path=get_kind_pv_bind_mount_path(volume_name)
@@ -403,7 +362,7 @@ class ClusterInfo:
             spec = client.V1PersistentVolumeSpec(
                 storage_class_name="manual",
                 access_modes=["ReadWriteOnce"],
-                capacity=to_k8s_resource_requirements(vol_resources).requests,
+                capacity=to_k8s_resource_requirements(resources).requests,
                 host_path=host_path,
             )
             pv = client.V1PersistentVolume(
@@ -449,29 +408,15 @@ class ClusterInfo:
         # 3. Fall back to spec.yml global (already resolved with DEFAULT fallback)
         return global_resources
 
-    def _build_containers(
-        self,
-        parsed_yaml_map: Any,
-        image_pull_policy: Optional[str] = None,
-    ) -> tuple:
-        """Build k8s container specs from parsed compose YAML.
-
-        Returns a tuple of (containers, init_containers, services, volumes)
-        where:
-        - containers: list of V1Container objects
-        - init_containers: list of V1Container objects for init containers
-          (compose services with label ``laconic.init-container: "true"``)
-        - services: the last services dict processed (used for annotations/labels)
-        - volumes: list of V1Volume objects
-        """
+    # TODO: put things like image pull policy into an object-scope struct
+    def get_deployment(self, image_pull_policy: str | None = None):
         containers = []
-        init_containers = []
         services = {}
         global_resources = self.spec.get_container_resources()
         if not global_resources:
             global_resources = DEFAULT_CONTAINER_RESOURCES
-        for pod_name in parsed_yaml_map:
-            pod = parsed_yaml_map[pod_name]
+        for pod_name in self.parsed_pod_yaml_map:
+            pod = self.parsed_pod_yaml_map[pod_name]
             services = pod["services"]
             for service_name in services:
                 container_name = service_name
@@ -492,9 +437,7 @@ class ClusterInfo:
                             port_str = port_str.split(":")[-1]
                         port = int(port_str)
                         container_ports.append(
-                            client.V1ContainerPort(
-                                container_port=port, protocol=protocol
-                            )
+                            client.V1ContainerPort(container_port=port, protocol=protocol)
                         )
                     if opts.o.debug:
                         print(f"image: {image}")
@@ -512,9 +455,7 @@ class ClusterInfo:
                 # Translate docker-compose service names to localhost for sidecars
                 # All services in the same pod share the network namespace
                 sibling_services = [s for s in services.keys() if s != service_name]
-                merged_envs = translate_sidecar_service_names(
-                    merged_envs, sibling_services
-                )
+                merged_envs = translate_sidecar_service_names(merged_envs, sibling_services)
                 envs = envs_from_environment_variables_map(merged_envs)
                 if opts.o.debug:
                     print(f"Merged envs: {envs}")
@@ -527,18 +468,14 @@ class ClusterInfo:
                     if self.spec.get_image_registry() is not None
                     else image
                 )
-                volume_mounts = volume_mounts_for_service(
-                    parsed_yaml_map, service_name
-                )
+                volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name)
                 # Handle command/entrypoint from compose file
                 # In docker-compose: entrypoint -> k8s command, command -> k8s args
                 container_command = None
                 container_args = None
                 if "entrypoint" in service_info:
                     entrypoint = service_info["entrypoint"]
-                    container_command = (
-                        entrypoint if isinstance(entrypoint, list) else [entrypoint]
-                    )
+                    container_command = entrypoint if isinstance(entrypoint, list) else [entrypoint]
                 if "command" in service_info:
                     cmd = service_info["command"]
                     container_args = cmd if isinstance(cmd, list) else cmd.split()
@@ -552,16 +489,6 @@ class ClusterInfo:
                         )
                     )
                 ]
-                # Mount user-declared secrets from spec.yml
-                for user_secret_name in self.spec.get_secrets():
-                    env_from.append(
-                        client.V1EnvFromSource(
-                            secret_ref=client.V1SecretEnvSource(
-                                name=user_secret_name,
-                                optional=True,
-                            )
-                        )
-                    )
                 container_resources = self._resolve_container_resources(
                     container_name, service_info, global_resources
                 )
@@ -577,40 +504,14 @@ class ClusterInfo:
                     volume_mounts=volume_mounts,
                     security_context=client.V1SecurityContext(
                         privileged=self.spec.get_privileged(),
-                        run_as_user=int(service_info["user"]) if "user" in service_info else None,
-                        capabilities=client.V1Capabilities(
-                            add=self.spec.get_capabilities()
-                        )
+                        capabilities=client.V1Capabilities(add=self.spec.get_capabilities())
                         if self.spec.get_capabilities()
                         else None,
                     ),
                     resources=to_k8s_resource_requirements(container_resources),
                 )
-                # Services with laconic.init-container label become
-                # k8s init containers instead of regular containers.
-                svc_labels = service_info.get("labels", {})
-                if isinstance(svc_labels, list):
-                    # docker-compose labels can be a list of "key=value"
-                    svc_labels = dict(
-                        item.split("=", 1) for item in svc_labels
-                    )
-                is_init = str(
-                    svc_labels.get("laconic.init-container", "")
-                ).lower() in ("true", "1", "yes")
-                if is_init:
-                    init_containers.append(container)
-                else:
-                    containers.append(container)
-        volumes = volumes_for_pod_files(
-            parsed_yaml_map, self.spec, self.app_name
-        )
-        return containers, init_containers, services, volumes
-
-    # TODO: put things like image pull policy into an object-scope struct
-    def get_deployment(self, image_pull_policy: Optional[str] = None):
-        containers, init_containers, services, volumes = self._build_containers(
-            self.parsed_pod_yaml_map, image_pull_policy
-        )
+                containers.append(container)
+        volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec, self.app_name)
         registry_config = self.spec.get_image_registry_config()
         if registry_config:
             secret_name = f"{self.app_name}-registry"
@@ -620,8 +521,6 @@ class ClusterInfo:
 
         annotations = None
         labels = {"app": self.app_name}
-        if self.stack_name:
-            labels["app.kubernetes.io/stack"] = self.stack_name
         affinity = None
         tolerations = None
 
@@ -679,7 +578,6 @@ class ClusterInfo:
             metadata=client.V1ObjectMeta(annotations=annotations, labels=labels),
             spec=client.V1PodSpec(
                 containers=containers,
-                init_containers=init_containers or None,
                 image_pull_secrets=image_pull_secrets,
                 volumes=volumes,
                 affinity=affinity,
@@ -698,83 +596,7 @@ class ClusterInfo:
         deployment = client.V1Deployment(
             api_version="apps/v1",
             kind="Deployment",
-            metadata=client.V1ObjectMeta(
-                name=f"{self.app_name}-deployment",
-                labels={"app": self.app_name, **({"app.kubernetes.io/stack": self.stack_name} if self.stack_name else {})},
-            ),
+            metadata=client.V1ObjectMeta(name=f"{self.app_name}-deployment"),
             spec=spec,
         )
         return deployment
-
-    def get_jobs(self, image_pull_policy: Optional[str] = None) -> List[client.V1Job]:
-        """Build k8s Job objects from parsed job compose files.
-
-        Each job compose file produces a V1Job with:
-        - restartPolicy: Never
-        - backoffLimit: 0
-        - Name: {app_name}-job-{job_name}
-        """
-        if not self.parsed_job_yaml_map:
-            return []
-
-        jobs = []
-        registry_config = self.spec.get_image_registry_config()
-        if registry_config:
-            secret_name = f"{self.app_name}-registry"
-            image_pull_secrets = [client.V1LocalObjectReference(name=secret_name)]
-        else:
-            image_pull_secrets = []
-
-        for job_file in self.parsed_job_yaml_map:
-            # Build containers for this single job file
-            single_job_map = {job_file: self.parsed_job_yaml_map[job_file]}
-            containers, init_containers, _services, volumes = (
-                self._build_containers(single_job_map, image_pull_policy)
-            )
-
-            # Derive job name from file path: docker-compose-<name>.yml -> <name>
-            base = os.path.basename(job_file)
-            # Strip docker-compose- prefix and .yml suffix
-            job_name = base
-            if job_name.startswith("docker-compose-"):
-                job_name = job_name[len("docker-compose-"):]
-            if job_name.endswith(".yml"):
-                job_name = job_name[: -len(".yml")]
-            elif job_name.endswith(".yaml"):
-                job_name = job_name[: -len(".yaml")]
-
-            # Use a distinct app label for job pods so they don't get
-            # picked up by pods_in_deployment() which queries app={app_name}.
-            pod_labels = {
-                "app": f"{self.app_name}-job",
-                **({"app.kubernetes.io/stack": self.stack_name} if self.stack_name else {}),
-            }
-            template = client.V1PodTemplateSpec(
-                metadata=client.V1ObjectMeta(
-                    labels=pod_labels
-                ),
-                spec=client.V1PodSpec(
-                    containers=containers,
-                    init_containers=init_containers or None,
-                    image_pull_secrets=image_pull_secrets,
-                    volumes=volumes,
-                    restart_policy="Never",
-                ),
-            )
-            job_spec = client.V1JobSpec(
-                template=template,
-                backoff_limit=0,
-            )
-            job_labels = {"app": self.app_name, **({"app.kubernetes.io/stack": self.stack_name} if self.stack_name else {})}
-            job = client.V1Job(
-                api_version="batch/v1",
-                kind="Job",
-                metadata=client.V1ObjectMeta(
-                    name=f"{self.app_name}-job-{job_name}",
-                    labels=job_labels,
-                ),
-                spec=job_spec,
-            )
-            jobs.append(job)
-
-        return jobs

stack_orchestrator/deploy/k8s/deploy_k8s.py

@@ -12,43 +12,41 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
+import time
 from datetime import datetime, timezone
-
 from pathlib import Path
+from typing import Any, cast
+
 from kubernetes import client, config
 from kubernetes.client.exceptions import ApiException
-from typing import Any, Dict, List, Optional, cast
 
 from stack_orchestrator import constants
 from stack_orchestrator.deploy.deployer import Deployer, DeployerConfigGenerator
+from stack_orchestrator.deploy.deployment_context import DeploymentContext
+from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo
 from stack_orchestrator.deploy.k8s.helpers import (
+    connect_registry_to_kind_network,
+    containers_in_pod,
     create_cluster,
     destroy_cluster,
-    load_images_into_kind,
-)
-from stack_orchestrator.deploy.k8s.helpers import (
-    install_ingress_for_kind,
-    wait_for_ingress_in_kind,
-    is_ingress_running,
-)
-from stack_orchestrator.deploy.k8s.helpers import (
-    pods_in_deployment,
-    containers_in_pod,
-    log_stream_from_string,
-)
-from stack_orchestrator.deploy.k8s.helpers import (
-    generate_kind_config,
+    ensure_local_registry,
     generate_high_memlock_spec_json,
+    generate_kind_config,
+    install_ingress_for_kind,
+    is_ingress_running,
+    local_registry_image,
+    log_stream_from_string,
+    pods_in_deployment,
+    push_images_to_local_registry,
+    wait_for_ingress_in_kind,
 )
-from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo
 from stack_orchestrator.opts import opts
-from stack_orchestrator.deploy.deployment_context import DeploymentContext
 from stack_orchestrator.util import error_exit
 
 
 class AttrDict(dict):
     def __init__(self, *args, **kwargs):
-        super(AttrDict, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.__dict__ = self
 
 
@@ -95,7 +93,6 @@ class K8sDeployer(Deployer):
     type: str
     core_api: client.CoreV1Api
     apps_api: client.AppsV1Api
-    batch_api: client.BatchV1Api
     networking_api: client.NetworkingV1Api
     k8s_namespace: str
     kind_cluster_name: str
@@ -111,7 +108,6 @@ class K8sDeployer(Deployer):
         compose_files,
         compose_project_name,
         compose_env_file,
-        job_compose_files=None,
     ) -> None:
         self.type = type
         self.skip_cluster_management = False
@@ -122,28 +118,19 @@ class K8sDeployer(Deployer):
             return
         self.deployment_dir = deployment_context.deployment_dir
         self.deployment_context = deployment_context
-        self.kind_cluster_name = deployment_context.spec.get_kind_cluster_name() or compose_project_name
-        # Use spec namespace if provided, otherwise derive from cluster-id
-        self.k8s_namespace = deployment_context.spec.get_namespace() or f"laconic-{compose_project_name}"
+        self.kind_cluster_name = compose_project_name
+        # Use deployment-specific namespace for resource isolation and easy cleanup
+        self.k8s_namespace = f"laconic-{compose_project_name}"
         self.cluster_info = ClusterInfo()
-        # stack.name may be an absolute path (from spec "stack:" key after
-        # path resolution). Extract just the directory basename for labels.
-        raw_name = deployment_context.stack.name if deployment_context else ""
-        stack_name = Path(raw_name).name if raw_name else ""
         self.cluster_info.int(
             compose_files,
             compose_env_file,
             compose_project_name,
             deployment_context.spec,
-            stack_name=stack_name,
         )
-        # Initialize job compose files if provided
-        if job_compose_files:
-            self.cluster_info.init_jobs(job_compose_files)
         if opts.o.debug:
             print(f"Deployment dir: {deployment_context.deployment_dir}")
             print(f"Compose files: {compose_files}")
-            print(f"Job compose files: {job_compose_files}")
             print(f"Project name: {compose_project_name}")
             print(f"Env file: {compose_env_file}")
             print(f"Type: {type}")
@@ -154,21 +141,28 @@ class K8sDeployer(Deployer):
         else:
             # Get the config file and pass to load_kube_config()
             config.load_kube_config(
-                config_file=self.deployment_dir.joinpath(
-                    constants.kube_config_filename
-                ).as_posix()
+                config_file=self.deployment_dir.joinpath(constants.kube_config_filename).as_posix()
             )
         self.core_api = client.CoreV1Api()
         self.networking_api = client.NetworkingV1Api()
         self.apps_api = client.AppsV1Api()
-        self.batch_api = client.BatchV1Api()
         self.custom_obj_api = client.CustomObjectsApi()
 
     def _ensure_namespace(self):
-        """Create the deployment namespace if it doesn't exist."""
+        """Create the deployment namespace if it doesn't exist.
+
+        If the namespace exists but is terminating (e.g., from a prior
+        down() call), wait for deletion to complete before creating a
+        fresh namespace.  K8s rejects resource creation in a terminating
+        namespace with 403 Forbidden, so proceeding without waiting
+        causes PVC/ConfigMap creation failures.
+        """
         if opts.o.dry_run:
             print(f"Dry run: would create namespace {self.k8s_namespace}")
             return
+
+        self._wait_for_namespace_deletion()
+
         try:
             self.core_api.read_namespace(name=self.k8s_namespace)
             if opts.o.debug:
@@ -188,6 +182,35 @@ class K8sDeployer(Deployer):
             else:
                 raise
 
+    def _wait_for_namespace_deletion(self):
+        """Block until the namespace is fully deleted, if it is terminating.
+
+        Polls every 2s for up to 120s.  If the namespace does not exist
+        (404) or is active, returns immediately.
+        """
+        deadline = time.monotonic() + 120
+        while True:
+            try:
+                ns = self.core_api.read_namespace(name=self.k8s_namespace)
+            except ApiException as e:
+                if e.status == 404:
+                    return  # Gone — ready to create
+                raise
+
+            phase = ns.status.phase if ns.status else None
+            if phase != "Terminating":
+                return  # Active or unknown — proceed
+
+            if time.monotonic() > deadline:
+                error_exit(
+                    f"Namespace {self.k8s_namespace} still terminating "
+                    f"after 120s — cannot proceed"
+                )
+
+            if opts.o.debug:
+                print(f"Namespace {self.k8s_namespace} is terminating, " f"waiting for deletion...")
+            time.sleep(2)
+
     def _delete_namespace(self):
         """Delete the deployment namespace and all resources within it."""
         if opts.o.dry_run:
@@ -204,92 +227,151 @@ class K8sDeployer(Deployer):
             else:
                 raise
 
-    def _delete_resources_by_label(self, label_selector: str, delete_volumes: bool):
-        """Delete only this stack's resources from a shared namespace."""
-        ns = self.k8s_namespace
-        if opts.o.dry_run:
-            print(f"Dry run: would delete resources with {label_selector} in {ns}")
+    def _ensure_config_map(self, cfg_map):
+        """Create or replace a ConfigMap (idempotent)."""
+        try:
+            resp = self.core_api.create_namespaced_config_map(
+                body=cfg_map, namespace=self.k8s_namespace
+            )
+            if opts.o.debug:
+                print(f"ConfigMap created: {resp}")
+        except ApiException as e:
+            if e.status == 409:
+                existing = self.core_api.read_namespaced_config_map(
+                    name=cfg_map.metadata.name, namespace=self.k8s_namespace
+                )
+                cfg_map.metadata.resource_version = existing.metadata.resource_version
+                resp = self.core_api.replace_namespaced_config_map(
+                    name=cfg_map.metadata.name,
+                    namespace=self.k8s_namespace,
+                    body=cfg_map,
+                )
+                if opts.o.debug:
+                    print(f"ConfigMap updated: {resp}")
+            else:
+                raise
+
+    def _ensure_deployment(self, deployment):
+        """Create or replace a Deployment (idempotent)."""
+        try:
+            resp = cast(
+                client.V1Deployment,
+                self.apps_api.create_namespaced_deployment(
+                    body=deployment, namespace=self.k8s_namespace
+                ),
+            )
+            if opts.o.debug:
+                print("Deployment created:")
+        except ApiException as e:
+            if e.status == 409:
+                existing = self.apps_api.read_namespaced_deployment(
+                    name=deployment.metadata.name,
+                    namespace=self.k8s_namespace,
+                )
+                deployment.metadata.resource_version = existing.metadata.resource_version
+                resp = cast(
+                    client.V1Deployment,
+                    self.apps_api.replace_namespaced_deployment(
+                        name=deployment.metadata.name,
+                        namespace=self.k8s_namespace,
+                        body=deployment,
+                    ),
+                )
+                if opts.o.debug:
+                    print("Deployment updated:")
+            else:
+                raise
+        if opts.o.debug:
+            meta = resp.metadata
+            spec = resp.spec
+            if meta and spec and spec.template.spec:
+                containers = spec.template.spec.containers
+                img = containers[0].image if containers else None
+                print(f"{meta.namespace} {meta.name} {meta.generation} {img}")
+
+    def _ensure_service(self, service, kind: str = "Service"):
+        """Create or replace a Service (idempotent).
+
+        Services have immutable fields (spec.clusterIP) that must be
+        preserved from the existing object on replace.
+        """
+        try:
+            resp = self.core_api.create_namespaced_service(
+                namespace=self.k8s_namespace, body=service
+            )
+            if opts.o.debug:
+                print(f"{kind} created: {resp}")
+        except ApiException as e:
+            if e.status == 409:
+                existing = self.core_api.read_namespaced_service(
+                    name=service.metadata.name, namespace=self.k8s_namespace
+                )
+                service.metadata.resource_version = existing.metadata.resource_version
+                if existing.spec.cluster_ip:
+                    service.spec.cluster_ip = existing.spec.cluster_ip
+                resp = self.core_api.replace_namespaced_service(
+                    name=service.metadata.name,
+                    namespace=self.k8s_namespace,
+                    body=service,
+                )
+                if opts.o.debug:
+                    print(f"{kind} updated: {resp}")
+            else:
+                raise
+
+    def _ensure_ingress(self, ingress):
+        """Create or replace an Ingress (idempotent)."""
+        try:
+            resp = self.networking_api.create_namespaced_ingress(
+                namespace=self.k8s_namespace, body=ingress
+            )
+            if opts.o.debug:
+                print(f"Ingress created: {resp}")
+        except ApiException as e:
+            if e.status == 409:
+                existing = self.networking_api.read_namespaced_ingress(
+                    name=ingress.metadata.name, namespace=self.k8s_namespace
+                )
+                ingress.metadata.resource_version = existing.metadata.resource_version
+                resp = self.networking_api.replace_namespaced_ingress(
+                    name=ingress.metadata.name,
+                    namespace=self.k8s_namespace,
+                    body=ingress,
+                )
+                if opts.o.debug:
+                    print(f"Ingress updated: {resp}")
+            else:
+                raise
+
+    def _clear_released_pv_claim_refs(self):
+        """Patch any Released PVs for this deployment to clear stale claimRefs.
+
+        After a namespace is deleted, PVCs are cascade-deleted but
+        cluster-scoped PVs survive in Released state with claimRefs
+        pointing to the now-deleted PVC UIDs.  New PVCs cannot bind
+        to these PVs until the stale claimRef is removed.
+        """
+        try:
+            pvs = self.core_api.list_persistent_volume(
+                label_selector=f"app={self.cluster_info.app_name}"
+            )
+        except ApiException:
             return
 
-        # Deployments
-        try:
-            deps = self.apps_api.list_namespaced_deployment(
-                namespace=ns, label_selector=label_selector
-            )
-            for dep in deps.items:
-                print(f"Deleting Deployment {dep.metadata.name}")
-                self.apps_api.delete_namespaced_deployment(
-                    name=dep.metadata.name, namespace=ns
-                )
-        except ApiException as e:
-            _check_delete_exception(e)
-
-        # Jobs
-        try:
-            jobs = self.batch_api.list_namespaced_job(
-                namespace=ns, label_selector=label_selector
-            )
-            for job in jobs.items:
-                print(f"Deleting Job {job.metadata.name}")
-                self.batch_api.delete_namespaced_job(
-                    name=job.metadata.name, namespace=ns,
-                    body=client.V1DeleteOptions(propagation_policy="Background"),
-                )
-        except ApiException as e:
-            _check_delete_exception(e)
-
-        # Services (NodePorts created by SO)
-        try:
-            svcs = self.core_api.list_namespaced_service(
-                namespace=ns, label_selector=label_selector
-            )
-            for svc in svcs.items:
-                print(f"Deleting Service {svc.metadata.name}")
-                self.core_api.delete_namespaced_service(
-                    name=svc.metadata.name, namespace=ns
-                )
-        except ApiException as e:
-            _check_delete_exception(e)
-
-        # Ingresses
-        try:
-            ings = self.networking_api.list_namespaced_ingress(
-                namespace=ns, label_selector=label_selector
-            )
-            for ing in ings.items:
-                print(f"Deleting Ingress {ing.metadata.name}")
-                self.networking_api.delete_namespaced_ingress(
-                    name=ing.metadata.name, namespace=ns
-                )
-        except ApiException as e:
-            _check_delete_exception(e)
-
-        # ConfigMaps
-        try:
-            cms = self.core_api.list_namespaced_config_map(
-                namespace=ns, label_selector=label_selector
-            )
-            for cm in cms.items:
-                print(f"Deleting ConfigMap {cm.metadata.name}")
-                self.core_api.delete_namespaced_config_map(
-                    name=cm.metadata.name, namespace=ns
-                )
-        except ApiException as e:
-            _check_delete_exception(e)
-
-        # PVCs (only if --delete-volumes)
-        if delete_volumes:
-            try:
-                pvcs = self.core_api.list_namespaced_persistent_volume_claim(
-                    namespace=ns, label_selector=label_selector
-                )
-                for pvc in pvcs.items:
-                    print(f"Deleting PVC {pvc.metadata.name}")
-                    self.core_api.delete_namespaced_persistent_volume_claim(
-                        name=pvc.metadata.name, namespace=ns
+        for pv in pvs.items:
+            phase = pv.status.phase if pv.status else None
+            if phase == "Released" and pv.spec and pv.spec.claim_ref:
+                pv_name = pv.metadata.name
+                if opts.o.debug:
+                    old_ref = pv.spec.claim_ref
+                    print(
+                        f"Clearing stale claimRef on PV {pv_name} "
+                        f"(was {old_ref.namespace}/{old_ref.name})"
                     )
-            except ApiException as e:
-                _check_delete_exception(e)
+                self.core_api.patch_persistent_volume(
+                    name=pv_name,
+                    body={"spec": {"claimRef": None}},
+                )
 
     def _create_volume_data(self):
         # Create the host-path-mounted PVs for this deployment
@@ -299,24 +381,29 @@ class K8sDeployer(Deployer):
                 print(f"Sending this pv: {pv}")
             if not opts.o.dry_run:
                 try:
-                    pv_resp = self.core_api.read_persistent_volume(
-                        name=pv.metadata.name
-                    )
+                    pv_resp = self.core_api.read_persistent_volume(name=pv.metadata.name)
                     if pv_resp:
                         if opts.o.debug:
                             print("PVs already present:")
                             print(f"{pv_resp}")
                         continue
-                except:  # noqa: E722
-                    pass
+                except ApiException as e:
+                    if e.status != 404:
+                        raise
 
                 pv_resp = self.core_api.create_persistent_volume(body=pv)
                 if opts.o.debug:
                     print("PVs created:")
                     print(f"{pv_resp}")
 
+        # After PV creation/verification, clear stale claimRefs on any
+        # Released PVs so that new PVCs can bind to them.
+        if not opts.o.dry_run:
+            self._clear_released_pv_claim_refs()
+
         # Figure out the PVCs for this deployment
         pvcs = self.cluster_info.get_pvcs()
+        pvc_errors = []
         for pvc in pvcs:
             if opts.o.debug:
                 print(f"Sending this pvc: {pvc}")
@@ -331,15 +418,27 @@ class K8sDeployer(Deployer):
                             print("PVCs already present:")
                             print(f"{pvc_resp}")
                         continue
-                except:  # noqa: E722
-                    pass
+                except ApiException as e:
+                    if e.status != 404:
+                        raise
 
-                pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(
-                    body=pvc, namespace=self.k8s_namespace
-                )
-                if opts.o.debug:
-                    print("PVCs created:")
-                    print(f"{pvc_resp}")
+                try:
+                    pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(
+                        body=pvc, namespace=self.k8s_namespace
+                    )
+                    if opts.o.debug:
+                        print("PVCs created:")
+                        print(f"{pvc_resp}")
+                except ApiException as e:
+                    pvc_name = pvc.metadata.name
+                    print(f"Error creating PVC {pvc_name}: {e.reason}")
+                    pvc_errors.append(pvc_name)
+
+        if pvc_errors:
+            error_exit(
+                f"Failed to create PVCs: {', '.join(pvc_errors)}. "
+                f"Check namespace state and PV availability."
+            )
 
         # Figure out the ConfigMaps for this deployment
         config_maps = self.cluster_info.get_configmaps()
@@ -347,75 +446,35 @@ class K8sDeployer(Deployer):
             if opts.o.debug:
                 print(f"Sending this ConfigMap: {cfg_map}")
             if not opts.o.dry_run:
-                cfg_rsp = self.core_api.create_namespaced_config_map(
-                    body=cfg_map, namespace=self.k8s_namespace
-                )
-                if opts.o.debug:
-                    print("ConfigMap created:")
-                    print(f"{cfg_rsp}")
+                self._ensure_config_map(cfg_map)
 
     def _create_deployment(self):
-        # Skip if there are no pods to deploy (e.g. jobs-only stacks)
-        if not self.cluster_info.parsed_pod_yaml_map:
-            if opts.o.debug:
-                print("No pods defined, skipping Deployment creation")
-            return
-        # Process compose files into a Deployment
+        """Create the k8s Deployment resource (which starts pods)."""
         deployment = self.cluster_info.get_deployment(
             image_pull_policy=None if self.is_kind() else "Always"
         )
-        # Create the k8s objects
+        if self.is_kind():
+            self._rewrite_local_images(deployment)
         if opts.o.debug:
             print(f"Sending this deployment: {deployment}")
         if not opts.o.dry_run:
-            deployment_resp = cast(
-                client.V1Deployment,
-                self.apps_api.create_namespaced_deployment(
-                    body=deployment, namespace=self.k8s_namespace
-                ),
-            )
-            if opts.o.debug:
-                print("Deployment created:")
-                meta = deployment_resp.metadata
-                spec = deployment_resp.spec
-                if meta and spec and spec.template.spec:
-                    ns = meta.namespace
-                    name = meta.name
-                    gen = meta.generation
-                    containers = spec.template.spec.containers
-                    img = containers[0].image if containers else None
-                    print(f"{ns} {name} {gen} {img}")
+            self._ensure_deployment(deployment)
 
-        service = self.cluster_info.get_service()
-        if opts.o.debug:
-            print(f"Sending this service: {service}")
-        if service and not opts.o.dry_run:
-            service_resp = self.core_api.create_namespaced_service(
-                namespace=self.k8s_namespace, body=service
-            )
-            if opts.o.debug:
-                print("Service created:")
-                print(f"{service_resp}")
+    def _rewrite_local_images(self, deployment):
+        """Rewrite local container images to use the local registry.
 
-    def _create_jobs(self):
-        # Process job compose files into k8s Jobs
-        jobs = self.cluster_info.get_jobs(
-            image_pull_policy=None if self.is_kind() else "Always"
-        )
-        for job in jobs:
-            if opts.o.debug:
-                print(f"Sending this job: {job}")
-            if not opts.o.dry_run:
-                job_resp = self.batch_api.create_namespaced_job(
-                    body=job, namespace=self.k8s_namespace
-                )
-                if opts.o.debug:
-                    print("Job created:")
-                    if job_resp.metadata:
-                        print(
-                            f"  {job_resp.metadata.namespace} "
-                            f"{job_resp.metadata.name}"
-                        )
+        Images built locally (listed in stack.yml containers) are pushed to
+        localhost:5001 by push_images_to_local_registry(). The k8s pod spec
+        must reference them at that address so containerd pulls from the
+        local registry instead of trying to find them in its local store.
+        """
+        local_containers = self.deployment_context.stack.obj.get("containers", [])
+        if not local_containers:
+            return
+        containers = deployment.spec.template.spec.containers or []
+        for container in containers:
+            if any(c in container.image for c in local_containers):
+                container.image = local_registry_image(container.image)
 
     def _find_certificate_for_host_name(self, host_name):
         all_certificates = self.custom_obj_api.list_namespaced_custom_object(
@@ -447,126 +506,119 @@ class K8sDeployer(Deployer):
                     if before < now < after:
                         # Check the status is Ready
                         for condition in status.get("conditions", []):
-                            if "True" == condition.get(
-                                "status"
-                            ) and "Ready" == condition.get("type"):
+                            if "True" == condition.get("status") and "Ready" == condition.get(
+                                "type"
+                            ):
                                 return cert
         return None
 
     def up(self, detach, skip_cluster_management, services):
+        self._setup_cluster_and_namespace(skip_cluster_management)
+        self._create_infrastructure()
+        self._create_deployment()
+
+    def _setup_cluster_and_namespace(self, skip_cluster_management):
+        """Create kind cluster (if needed) and namespace.
+
+        Shared by up() and prepare().
+        """
         self.skip_cluster_management = skip_cluster_management
         if not opts.o.dry_run:
             if self.is_kind() and not self.skip_cluster_management:
-                # Create the kind cluster (or reuse existing one)
-                kind_config = str(
-                    self.deployment_dir.joinpath(constants.kind_config_filename)
-                )
+                ensure_local_registry()
+                kind_config = str(self.deployment_dir.joinpath(constants.kind_config_filename))
                 actual_cluster = create_cluster(self.kind_cluster_name, kind_config)
                 if actual_cluster != self.kind_cluster_name:
-                    # An existing cluster was found, use it instead
                     self.kind_cluster_name = actual_cluster
-                # Only load locally-built images into kind
-                # Registry images (docker.io, ghcr.io, etc.) will be pulled by k8s
-                local_containers = self.deployment_context.stack.obj.get(
-                    "containers", []
-                )
+                connect_registry_to_kind_network(self.kind_cluster_name)
+                local_containers = self.deployment_context.stack.obj.get("containers", [])
                 if local_containers:
-                    # Filter image_set to only images matching local containers
                     local_images = {
                         img
                         for img in self.cluster_info.image_set
                         if any(c in img for c in local_containers)
                     }
                     if local_images:
-                        load_images_into_kind(self.kind_cluster_name, local_images)
-                # Note: if no local containers defined, all images come from registries
+                        push_images_to_local_registry(local_images)
             self.connect_api()
-            # Create deployment-specific namespace for resource isolation
             self._ensure_namespace()
             if self.is_kind() and not self.skip_cluster_management:
-                # Configure ingress controller (not installed by default in kind)
-                # Skip if already running (idempotent for shared cluster)
                 if not is_ingress_running():
                     install_ingress_for_kind(self.cluster_info.spec.get_acme_email())
-                    # Wait for ingress to start
-                    # (deployment provisioning will fail unless this is done)
                     wait_for_ingress_in_kind()
-                # Create RuntimeClass if unlimited_memlock is enabled
                 if self.cluster_info.spec.get_unlimited_memlock():
                     _create_runtime_class(
                         constants.high_memlock_runtime,
                         constants.high_memlock_runtime,
                     )
-
         else:
             print("Dry run mode enabled, skipping k8s API connect")
 
-        # Create registry secret if configured
+    def _create_infrastructure(self):
+        """Create PVs, PVCs, ConfigMaps, Services, Ingresses, NodePorts.
+
+        Everything except the Deployment resource (which starts pods).
+        Shared by up() and prepare().
+        """
         from stack_orchestrator.deploy.deployment_create import create_registry_secret
 
         create_registry_secret(self.cluster_info.spec, self.cluster_info.app_name)
 
         self._create_volume_data()
-        self._create_deployment()
-        self._create_jobs()
+
+        # Create the ClusterIP service (paired with the deployment)
+        service = self.cluster_info.get_service()
+        if service and not opts.o.dry_run:
+            if opts.o.debug:
+                print(f"Sending this service: {service}")
+            self._ensure_service(service)
 
         http_proxy_info = self.cluster_info.spec.get_http_proxy()
-        # Note: we don't support tls for kind (enabling tls causes errors)
         use_tls = http_proxy_info and not self.is_kind()
         certificate = (
             self._find_certificate_for_host_name(http_proxy_info[0]["host-name"])
             if use_tls
             else None
         )
-        if opts.o.debug:
-            if certificate:
-                print(f"Using existing certificate: {certificate}")
+        if opts.o.debug and certificate:
+            print(f"Using existing certificate: {certificate}")
 
-        ingress = self.cluster_info.get_ingress(
-            use_tls=use_tls, certificate=certificate
-        )
+        ingress = self.cluster_info.get_ingress(use_tls=use_tls, certificate=certificate)
         if ingress:
             if opts.o.debug:
                 print(f"Sending this ingress: {ingress}")
             if not opts.o.dry_run:
-                ingress_resp = self.networking_api.create_namespaced_ingress(
-                    namespace=self.k8s_namespace, body=ingress
-                )
-                if opts.o.debug:
-                    print("Ingress created:")
-                    print(f"{ingress_resp}")
-        else:
-            if opts.o.debug:
-                print("No ingress configured")
+                self._ensure_ingress(ingress)
+        elif opts.o.debug:
+            print("No ingress configured")
 
-        nodeports: List[client.V1Service] = self.cluster_info.get_nodeports()
+        nodeports: list[client.V1Service] = self.cluster_info.get_nodeports()
         for nodeport in nodeports:
             if opts.o.debug:
                 print(f"Sending this nodeport: {nodeport}")
             if not opts.o.dry_run:
-                nodeport_resp = self.core_api.create_namespaced_service(
-                    namespace=self.k8s_namespace, body=nodeport
-                )
-                if opts.o.debug:
-                    print("NodePort created:")
-                    print(f"{nodeport_resp}")
+                self._ensure_service(nodeport, kind="NodePort")
 
-        # Call start() hooks — stacks can create additional k8s resources
-        if self.deployment_context:
-            from stack_orchestrator.deploy.deployment_create import call_stack_deploy_start
-            call_stack_deploy_start(self.deployment_context)
+    def prepare(self, skip_cluster_management):
+        """Create cluster infrastructure without starting pods.
+
+        Sets up kind cluster, namespace, PVs, PVCs, ConfigMaps, Services,
+        Ingresses, and NodePorts — everything that up() does EXCEPT creating
+        the Deployment resource.
+        """
+        self._setup_cluster_and_namespace(skip_cluster_management)
+        self._create_infrastructure()
+        print("Cluster infrastructure prepared (no pods started).")
 
     def down(self, timeout, volumes, skip_cluster_management):
         self.skip_cluster_management = skip_cluster_management
         self.connect_api()
 
-        app_label = f"app={self.cluster_info.app_name}"
-
         # PersistentVolumes are cluster-scoped (not namespaced), so delete by label
         if volumes:
             try:
                 pvs = self.core_api.list_persistent_volume(
-                    label_selector=app_label
+                    label_selector=f"app={self.cluster_info.app_name}"
                 )
                 for pv in pvs.items:
                     if opts.o.debug:
@@ -579,14 +631,9 @@ class K8sDeployer(Deployer):
                 if opts.o.debug:
                     print(f"Error listing PVs: {e}")
 
-        # When namespace is explicitly set in the spec, it may be shared with
-        # other stacks — delete only this stack's resources by label.
-        # Otherwise the namespace is owned by this deployment, delete it entirely.
-        shared_namespace = self.deployment_context.spec.get_namespace() is not None
-        if shared_namespace:
-            self._delete_resources_by_label(app_label, volumes)
-        else:
-            self._delete_namespace()
+        # Delete the deployment namespace - this cascades to all namespaced resources
+        # (PVCs, ConfigMaps, Deployments, Services, Ingresses, etc.)
+        self._delete_namespace()
 
         if self.is_kind() and not self.skip_cluster_management:
             # Destroy the kind cluster
@@ -625,7 +672,7 @@ class K8sDeployer(Deployer):
                 return
 
             cert = cast(
-                Dict[str, Any],
+                dict[str, Any],
                 self.custom_obj_api.get_namespaced_custom_object(
                     group="cert-manager.io",
                     version="v1",
@@ -641,7 +688,7 @@ class K8sDeployer(Deployer):
                 if lb_ingress:
                     ip = lb_ingress[0].ip or "?"
             cert_status = cert.get("status", {})
-            tls = "notBefore: %s; notAfter: %s; names: %s" % (
+            tls = "notBefore: {}; notAfter: {}; names: {}".format(
                 cert_status.get("notBefore", "?"),
                 cert_status.get("notAfter", "?"),
                 ingress.spec.tls[0].hosts,
@@ -682,9 +729,7 @@ class K8sDeployer(Deployer):
                     if c.ports:
                         for prt in c.ports:
                             ports[str(prt.container_port)] = [
-                                AttrDict(
-                                    {"HostIp": pod_ip, "HostPort": prt.container_port}
-                                )
+                                AttrDict({"HostIp": pod_ip, "HostPort": prt.container_port})
                             ]
 
                 ret.append(
@@ -711,14 +756,14 @@ class K8sDeployer(Deployer):
 
     def logs(self, services, tail, follow, stream):
         self.connect_api()
-        pods = pods_in_deployment(self.core_api, self.cluster_info.app_name, namespace=self.k8s_namespace)
+        pods = pods_in_deployment(self.core_api, self.cluster_info.app_name)
         if len(pods) > 1:
             print("Warning: more than one pod in the deployment")
         if len(pods) == 0:
             log_data = "******* Pods not running ********\n"
         else:
             k8s_pod_name = pods[0]
-            containers = containers_in_pod(self.core_api, k8s_pod_name, namespace=self.k8s_namespace)
+            containers = containers_in_pod(self.core_api, k8s_pod_name)
             # If pod not started, logs request below will throw an exception
             try:
                 log_data = ""
@@ -735,11 +780,7 @@ class K8sDeployer(Deployer):
                 log_data = "******* No logs available ********\n"
         return log_stream_from_string(log_data)
 
-    def update(self):
-        if not self.cluster_info.parsed_pod_yaml_map:
-            if opts.o.debug:
-                print("No pods defined, skipping update")
-            return
+    def update_envs(self):
         self.connect_api()
         ref_deployment = self.cluster_info.get_deployment()
         if not ref_deployment or not ref_deployment.metadata:
@@ -750,9 +791,7 @@ class K8sDeployer(Deployer):
 
         deployment = cast(
             client.V1Deployment,
-            self.apps_api.read_namespaced_deployment(
-                name=ref_name, namespace=self.k8s_namespace
-            ),
+            self.apps_api.read_namespaced_deployment(name=ref_name, namespace=self.k8s_namespace),
         )
         if not deployment.spec or not deployment.spec.template:
             return
@@ -791,52 +830,35 @@ class K8sDeployer(Deployer):
         user=None,
         volumes=None,
         entrypoint=None,
-        env={},
-        ports=[],
+        env=None,
+        ports=None,
         detach=False,
     ):
         # We need to figure out how to do this -- check why we're being called first
         pass
 
-    def run_job(self, job_name: str, helm_release: Optional[str] = None):
+    def run_job(self, job_name: str, helm_release: str | None = None):
         if not opts.o.dry_run:
+            from stack_orchestrator.deploy.k8s.helm.job_runner import run_helm_job
+
             # Check if this is a helm-based deployment
             chart_dir = self.deployment_dir / "chart"
-            if chart_dir.exists():
-                from stack_orchestrator.deploy.k8s.helm.job_runner import run_helm_job
+            if not chart_dir.exists():
+                # TODO: Implement job support for compose-based K8s deployments
+                raise Exception(
+                    f"Job support is only available for helm-based "
+                    f"deployments. Chart directory not found: {chart_dir}"
+                )
 
-                # Run the job using the helm job runner
-                run_helm_job(
-                    chart_dir=chart_dir,
-                    job_name=job_name,
-                    release=helm_release,
-                    namespace=self.k8s_namespace,
-                    timeout=600,
-                    verbose=opts.o.verbose,
-                )
-            else:
-                # Non-Helm path: create job from ClusterInfo
-                self.connect_api()
-                jobs = self.cluster_info.get_jobs(
-                    image_pull_policy=None if self.is_kind() else "Always"
-                )
-                # Find the matching job by name
-                target_name = f"{self.cluster_info.app_name}-job-{job_name}"
-                matched_job = None
-                for job in jobs:
-                    if job.metadata and job.metadata.name == target_name:
-                        matched_job = job
-                        break
-                if matched_job is None:
-                    raise Exception(
-                        f"Job '{job_name}' not found. Available jobs: "
-                        f"{[j.metadata.name for j in jobs if j.metadata]}"
-                    )
-                if opts.o.debug:
-                    print(f"Creating job: {target_name}")
-                self.batch_api.create_namespaced_job(
-                    body=matched_job, namespace=self.k8s_namespace
-                )
+            # Run the job using the helm job runner
+            run_helm_job(
+                chart_dir=chart_dir,
+                job_name=job_name,
+                release=helm_release,
+                namespace=self.k8s_namespace,
+                timeout=600,
+                verbose=opts.o.verbose,
+            )
 
     def is_kind(self):
         return self.type == "k8s-kind"
@@ -857,13 +879,9 @@ class K8sDeployerConfigGenerator(DeployerConfigGenerator):
             # Must be done before generate_kind_config() which references it.
             if self.deployment_context.spec.get_unlimited_memlock():
                 spec_content = generate_high_memlock_spec_json()
-                spec_file = deployment_dir.joinpath(
-                    constants.high_memlock_spec_filename
-                )
+                spec_file = deployment_dir.joinpath(constants.high_memlock_spec_filename)
                 if opts.o.debug:
-                    print(
-                        f"Creating high-memlock spec for unlimited memlock: {spec_file}"
-                    )
+                    print(f"Creating high-memlock spec for unlimited memlock: {spec_file}")
                 with open(spec_file, "w") as output_file:
                     output_file.write(spec_content)
 

stack_orchestrator/deploy/k8s/helm/chart_generator.py

@@ -16,21 +16,21 @@
 from pathlib import Path
 
 from stack_orchestrator import constants
-from stack_orchestrator.opts import opts
-from stack_orchestrator.util import (
-    get_parsed_stack_config,
-    get_pod_list,
-    get_pod_file_path,
-    get_job_list,
-    get_job_file_path,
-    error_exit,
-)
 from stack_orchestrator.deploy.k8s.helm.kompose_wrapper import (
     check_kompose_available,
-    get_kompose_version,
     convert_to_helm_chart,
+    get_kompose_version,
+)
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import (
+    error_exit,
+    get_job_file_path,
+    get_job_list,
+    get_parsed_stack_config,
+    get_pod_file_path,
+    get_pod_list,
+    get_yaml,
 )
-from stack_orchestrator.util import get_yaml
 
 
 def _wrap_job_templates_with_conditionals(chart_dir: Path, jobs: list) -> None:
@@ -88,7 +88,7 @@ def _post_process_chart(chart_dir: Path, chart_name: str, jobs: list) -> None:
     # Fix Chart.yaml
     chart_yaml_path = chart_dir / "Chart.yaml"
     if chart_yaml_path.exists():
-        chart_yaml = yaml.load(open(chart_yaml_path, "r"))
+        chart_yaml = yaml.load(open(chart_yaml_path))
 
         # Fix name
         chart_yaml["name"] = chart_name
@@ -108,9 +108,7 @@ def _post_process_chart(chart_dir: Path, chart_name: str, jobs: list) -> None:
         _wrap_job_templates_with_conditionals(chart_dir, jobs)
 
 
-def generate_helm_chart(
-    stack_path: str, spec_file: str, deployment_dir_path: Path
-) -> None:
+def generate_helm_chart(stack_path: str, spec_file: str, deployment_dir_path: Path) -> None:
     """
     Generate a self-sufficient Helm chart from stack compose files using Kompose.
 
@@ -152,7 +150,7 @@ def generate_helm_chart(
         error_exit(f"Deployment file not found: {deployment_file}")
 
     yaml = get_yaml()
-    deployment_config = yaml.load(open(deployment_file, "r"))
+    deployment_config = yaml.load(open(deployment_file))
     cluster_id = deployment_config.get(constants.cluster_id_key)
     if not cluster_id:
         error_exit(f"cluster-id not found in {deployment_file}")
@@ -219,10 +217,7 @@ def generate_helm_chart(
     # 5. Create chart directory and invoke Kompose
     chart_dir = deployment_dir_path / "chart"
 
-    print(
-        f"Converting {len(compose_files)} compose file(s) to Helm chart "
-        "using Kompose..."
-    )
+    print(f"Converting {len(compose_files)} compose file(s) to Helm chart " "using Kompose...")
 
     try:
         output = convert_to_helm_chart(
@@ -304,9 +299,7 @@ Edit the generated template files in `templates/` to customize:
 
     # Count generated files
     template_files = (
-        list((chart_dir / "templates").glob("*.yaml"))
-        if (chart_dir / "templates").exists()
-        else []
+        list((chart_dir / "templates").glob("*.yaml")) if (chart_dir / "templates").exists() else []
     )
     print(f"  Files:    {len(template_files)} template(s) generated")
 

stack_orchestrator/deploy/k8s/helm/job_runner.py

@@ -13,12 +13,12 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
+import json
+import os
 import subprocess
 import tempfile
-import os
-import json
 from pathlib import Path
-from typing import Optional
+
 from stack_orchestrator.util import get_yaml
 
 
@@ -40,18 +40,19 @@ def get_release_name_from_chart(chart_dir: Path) -> str:
         raise Exception(f"Chart.yaml not found: {chart_yaml_path}")
 
     yaml = get_yaml()
-    chart_yaml = yaml.load(open(chart_yaml_path, "r"))
+    chart_yaml = yaml.load(open(chart_yaml_path))
 
     if "name" not in chart_yaml:
         raise Exception(f"Chart name not found in {chart_yaml_path}")
 
-    return chart_yaml["name"]
+    name: str = chart_yaml["name"]
+    return name
 
 
 def run_helm_job(
     chart_dir: Path,
     job_name: str,
-    release: Optional[str] = None,
+    release: str | None = None,
     namespace: str = "default",
     timeout: int = 600,
     verbose: bool = False,
@@ -94,9 +95,7 @@ def run_helm_job(
         print(f"Running job '{job_name}' from helm chart: {chart_dir}")
 
     # Use helm template to render the job manifest
-    with tempfile.NamedTemporaryFile(
-        mode="w", suffix=".yaml", delete=False
-    ) as tmp_file:
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as tmp_file:
         try:
             # Render job template with job enabled
             # Use --set-json to properly handle job names with dashes
@@ -116,9 +115,7 @@ def run_helm_job(
             if verbose:
                 print(f"Running: {' '.join(helm_cmd)}")
 
-            result = subprocess.run(
-                helm_cmd, check=True, capture_output=True, text=True
-            )
+            result = subprocess.run(helm_cmd, check=True, capture_output=True, text=True)
             tmp_file.write(result.stdout)
             tmp_file.flush()
 
@@ -139,9 +136,7 @@ def run_helm_job(
                 "-n",
                 namespace,
             ]
-            subprocess.run(
-                kubectl_apply_cmd, check=True, capture_output=True, text=True
-            )
+            subprocess.run(kubectl_apply_cmd, check=True, capture_output=True, text=True)
 
             if verbose:
                 print(f"Job {actual_job_name} created, waiting for completion...")
@@ -164,7 +159,7 @@ def run_helm_job(
 
         except subprocess.CalledProcessError as e:
             error_msg = e.stderr if e.stderr else str(e)
-            raise Exception(f"Job failed: {error_msg}")
+            raise Exception(f"Job failed: {error_msg}") from e
         finally:
             # Clean up temp file
             if os.path.exists(tmp_file.name):

stack_orchestrator/deploy/k8s/helm/kompose_wrapper.py

@@ -13,10 +13,9 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-import subprocess
 import shutil
+import subprocess
 from pathlib import Path
-from typing import List, Optional
 
 
 def check_kompose_available() -> bool:
@@ -37,9 +36,7 @@ def get_kompose_version() -> str:
     if not check_kompose_available():
         raise Exception("kompose not found in PATH")
 
-    result = subprocess.run(
-        ["kompose", "version"], capture_output=True, text=True, timeout=10
-    )
+    result = subprocess.run(["kompose", "version"], capture_output=True, text=True, timeout=10)
 
     if result.returncode != 0:
         raise Exception(f"Failed to get kompose version: {result.stderr}")
@@ -53,7 +50,7 @@ def get_kompose_version() -> str:
 
 
 def convert_to_helm_chart(
-    compose_files: List[Path], output_dir: Path, chart_name: Optional[str] = None
+    compose_files: list[Path], output_dir: Path, chart_name: str | None = None
 ) -> str:
     """
     Invoke kompose to convert Docker Compose files to a Helm chart.
@@ -71,8 +68,7 @@ def convert_to_helm_chart(
     """
     if not check_kompose_available():
         raise Exception(
-            "kompose not found in PATH. "
-            "Install from: https://kompose.io/installation/"
+            "kompose not found in PATH. " "Install from: https://kompose.io/installation/"
         )
 
     # Ensure output directory exists
@@ -95,9 +91,7 @@ def convert_to_helm_chart(
 
     if result.returncode != 0:
         raise Exception(
-            f"Kompose conversion failed:\n"
-            f"Command: {' '.join(cmd)}\n"
-            f"Error: {result.stderr}"
+            f"Kompose conversion failed:\n" f"Command: {' '.join(cmd)}\n" f"Error: {result.stderr}"
         )
 
     return result.stdout

stack_orchestrator/deploy/k8s/helpers.py

@@ -13,20 +13,22 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
+import os
+import re
+import subprocess
+from collections.abc import Mapping
+from pathlib import Path
+from typing import cast
+
+import yaml
 from kubernetes import client, utils, watch
 from kubernetes.client.exceptions import ApiException
-import os
-from pathlib import Path
-import subprocess
-import re
-from typing import Set, Mapping, List, Optional, cast
-import yaml
 
-from stack_orchestrator.util import get_k8s_dir, error_exit
-from stack_orchestrator.opts import opts
+from stack_orchestrator import constants
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names
 from stack_orchestrator.deploy.deployer import DeployerException
-from stack_orchestrator import constants
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import error_exit, get_k8s_dir
 
 
 def is_host_path_mount(volume_name: str) -> bool:
@@ -77,9 +79,7 @@ def get_kind_cluster():
     Uses `kind get clusters` to find existing clusters.
     Returns the cluster name or None if no cluster exists.
     """
-    result = subprocess.run(
-        "kind get clusters", shell=True, capture_output=True, text=True
-    )
+    result = subprocess.run("kind get clusters", shell=True, capture_output=True, text=True)
     if result.returncode != 0:
         return None
 
@@ -98,12 +98,12 @@ def _run_command(command: str):
     return result
 
 
-def _get_etcd_host_path_from_kind_config(config_file: str) -> Optional[str]:
+def _get_etcd_host_path_from_kind_config(config_file: str) -> str | None:
     """Extract etcd host path from kind config extraMounts."""
     import yaml
 
     try:
-        with open(config_file, "r") as f:
+        with open(config_file) as f:
             config = yaml.safe_load(f)
     except Exception:
         return None
@@ -113,7 +113,8 @@ def _get_etcd_host_path_from_kind_config(config_file: str) -> Optional[str]:
         extra_mounts = node.get("extraMounts", [])
         for mount in extra_mounts:
             if mount.get("containerPath") == "/var/lib/etcd":
-                return mount.get("hostPath")
+                host_path: str | None = mount.get("hostPath")
+                return host_path
     return None
 
 
@@ -133,8 +134,7 @@ def _clean_etcd_keeping_certs(etcd_path: str) -> bool:
     db_path = Path(etcd_path) / "member" / "snap" / "db"
     # Check existence using docker since etcd dir is root-owned
     check_cmd = (
-        f"docker run --rm -v {etcd_path}:/etcd:ro alpine:3.19 "
-        "test -f /etcd/member/snap/db"
+        f"docker run --rm -v {etcd_path}:/etcd:ro alpine:3.19 " "test -f /etcd/member/snap/db"
     )
     check_result = subprocess.run(check_cmd, shell=True, capture_output=True)
     if check_result.returncode != 0:
@@ -148,8 +148,16 @@ def _clean_etcd_keeping_certs(etcd_path: str) -> bool:
     etcd_image = "gcr.io/etcd-development/etcd:v3.5.9"
     temp_dir = "/tmp/laconic-etcd-cleanup"
 
-    # Whitelist: prefixes to KEEP - everything else gets deleted
-    keep_prefixes = "/registry/secrets/caddy-system"
+    # Whitelist: prefixes to KEEP - everything else gets deleted.
+    # Must include core cluster resources (kubernetes service, kube-system
+    # secrets) or kindnet panics on restart — KUBERNETES_SERVICE_HOST is
+    # injected from the kubernetes ClusterIP service in default namespace.
+    keep_prefixes = [
+        "/registry/secrets/caddy-system",
+        "/registry/services/specs/default/kubernetes",
+        "/registry/services/endpoints/default/kubernetes",
+    ]
+    keep_prefixes_str = " ".join(keep_prefixes)
 
     # The etcd image is distroless (no shell). We extract the statically-linked
     # etcdctl binary and run it from alpine which has shell + jq support.
@@ -195,13 +203,21 @@ def _clean_etcd_keeping_certs(etcd_path: str) -> bool:
         sleep 3
 
         # Use alpine with extracted etcdctl to run commands (alpine has shell + jq)
-        # Export caddy secrets
+        # Export whitelisted keys (caddy TLS certs + core cluster services)
         docker run --rm \
             -v {temp_dir}:/backup \
             --network container:laconic-etcd-cleanup \
-            $ALPINE_IMAGE sh -c \
-            '/backup/etcdctl get --prefix "{keep_prefixes}" -w json \
-                > /backup/kept.json 2>/dev/null || echo "{{}}" > /backup/kept.json'
+            $ALPINE_IMAGE sh -c '
+                apk add --no-cache jq >/dev/null 2>&1
+                echo "[]" > /backup/all-kvs.json
+                for prefix in {keep_prefixes_str}; do
+                    /backup/etcdctl get --prefix "$prefix" -w json 2>/dev/null \
+                        | jq ".kvs // []" >> /backup/all-kvs.json || true
+                done
+                jq -s "add" /backup/all-kvs.json \
+                    | jq "{{kvs: .}}" > /backup/kept.json 2>/dev/null \
+                    || echo "{{}}" > /backup/kept.json
+            '
 
         # Delete ALL registry keys
         docker run --rm \
@@ -321,7 +337,7 @@ def is_ingress_running() -> bool:
 
 def wait_for_ingress_in_kind():
     core_v1 = client.CoreV1Api()
-    for i in range(20):
+    for _i in range(20):
         warned_waiting = False
         w = watch.Watch()
         for event in w.stream(
@@ -348,9 +364,7 @@ def wait_for_ingress_in_kind():
 def install_ingress_for_kind(acme_email: str = ""):
     api_client = client.ApiClient()
     ingress_install = os.path.abspath(
-        get_k8s_dir().joinpath(
-            "components", "ingress", "ingress-caddy-kind-deploy.yaml"
-        )
+        get_k8s_dir().joinpath("components", "ingress", "ingress-caddy-kind-deploy.yaml")
     )
     if opts.o.debug:
         print("Installing Caddy ingress controller in kind cluster")
@@ -384,19 +398,88 @@ def install_ingress_for_kind(acme_email: str = ""):
         )
 
 
-def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]):
-    for image in image_set:
+LOCAL_REGISTRY_NAME = "kind-registry"
+LOCAL_REGISTRY_HOST_PORT = 5001
+LOCAL_REGISTRY_CONTAINER_PORT = 5000
+
+
+def ensure_local_registry():
+    """Ensure a persistent local registry container is running.
+
+    The registry survives kind cluster recreates — images pushed to it
+    remain available without re-pushing. After ensuring the registry is
+    running, connects it to the kind Docker network so kind nodes can
+    pull from it.
+    """
+    # Check if registry container exists (running or stopped)
+    check = subprocess.run(
+        f"docker inspect {LOCAL_REGISTRY_NAME}",
+        shell=True,
+        capture_output=True,
+    )
+    if check.returncode != 0:
+        # Create the registry container
         result = _run_command(
-            f"kind load docker-image {image} --name {kind_cluster_name}"
+            f"docker run -d --restart=always"
+            f" -p {LOCAL_REGISTRY_HOST_PORT}:{LOCAL_REGISTRY_CONTAINER_PORT}"
+            f" --name {LOCAL_REGISTRY_NAME} registry:2"
         )
         if result.returncode != 0:
-            raise DeployerException(f"kind load docker-image failed: {result}")
+            raise DeployerException(f"Failed to start local registry: {result}")
+        print(f"Started local registry on port {LOCAL_REGISTRY_HOST_PORT}")
+    else:
+        # Ensure it's running (may have been stopped)
+        _run_command(f"docker start {LOCAL_REGISTRY_NAME}")
+        if opts.o.debug:
+            print("Local registry already exists, ensured running")
 
 
-def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str, namespace: str = "default"):
+def connect_registry_to_kind_network(kind_cluster_name: str):
+    """Connect the local registry to the kind Docker network.
+
+    Idempotent — silently succeeds if already connected.
+    """
+    network = "kind"
+    result = subprocess.run(
+        f"docker network connect {network} {LOCAL_REGISTRY_NAME}",
+        shell=True,
+        capture_output=True,
+    )
+    if result.returncode != 0 and b"already exists" not in result.stderr:
+        raise DeployerException(
+            f"Failed to connect registry to kind network: " f"{result.stderr.decode()}"
+        )
+
+
+def push_images_to_local_registry(image_set: set[str]):
+    """Tag and push images to the local registry.
+
+    Near-instant compared to kind load (shared filesystem, layer dedup).
+    """
+    for image in image_set:
+        registry_image = local_registry_image(image)
+        tag_result = _run_command(f"docker tag {image} {registry_image}")
+        if tag_result.returncode != 0:
+            raise DeployerException(f"docker tag failed for {image}: {tag_result}")
+        push_result = _run_command(f"docker push {registry_image}")
+        if push_result.returncode != 0:
+            raise DeployerException(f"docker push failed for {registry_image}: {push_result}")
+        if opts.o.debug:
+            print(f"Pushed {registry_image} to local registry")
+
+
+def local_registry_image(image: str) -> str:
+    """Rewrite an image reference to use the local registry.
+
+    e.g. laconicnetwork/agave:local -> localhost:5001/laconicnetwork/agave:local
+    """
+    return f"localhost:{LOCAL_REGISTRY_HOST_PORT}/{image}"
+
+
+def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
     pods = []
     pod_response = core_api.list_namespaced_pod(
-        namespace=namespace, label_selector=f"app={deployment_name}"
+        namespace="default", label_selector=f"app={deployment_name}"
     )
     if opts.o.debug:
         print(f"pod_response: {pod_response}")
@@ -406,11 +489,9 @@ def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str, namespa
     return pods
 
 
-def containers_in_pod(core_api: client.CoreV1Api, pod_name: str, namespace: str = "default") -> List[str]:
-    containers: List[str] = []
-    pod_response = cast(
-        client.V1Pod, core_api.read_namespaced_pod(pod_name, namespace=namespace)
-    )
+def containers_in_pod(core_api: client.CoreV1Api, pod_name: str) -> list[str]:
+    containers: list[str] = []
+    pod_response = cast(client.V1Pod, core_api.read_namespaced_pod(pod_name, namespace="default"))
     if opts.o.debug:
         print(f"pod_response: {pod_response}")
     if not pod_response.spec or not pod_response.spec.containers:
@@ -433,7 +514,7 @@ def named_volumes_from_pod_files(parsed_pod_files):
         parsed_pod_file = parsed_pod_files[pod]
         if "volumes" in parsed_pod_file:
             volumes = parsed_pod_file["volumes"]
-            for volume, value in volumes.items():
+            for volume, _value in volumes.items():
                 # Volume definition looks like:
                 # 'laconicd-data': None
                 named_volumes.append(volume)
@@ -465,14 +546,10 @@ def volume_mounts_for_service(parsed_pod_files, service):
                             mount_split = mount_string.split(":")
                             volume_name = mount_split[0]
                             mount_path = mount_split[1]
-                            mount_options = (
-                                mount_split[2] if len(mount_split) == 3 else None
-                            )
+                            mount_options = mount_split[2] if len(mount_split) == 3 else None
                             # For host path mounts, use sanitized name
                             if is_host_path_mount(volume_name):
-                                k8s_volume_name = sanitize_host_path_to_volume_name(
-                                    volume_name
-                                )
+                                k8s_volume_name = sanitize_host_path_to_volume_name(volume_name)
                             else:
                                 k8s_volume_name = volume_name
                             if opts.o.debug:
@@ -511,9 +588,7 @@ def volumes_for_pod_files(parsed_pod_files, spec, app_name):
                     claim = client.V1PersistentVolumeClaimVolumeSource(
                         claim_name=f"{app_name}-{volume_name}"
                     )
-                    volume = client.V1Volume(
-                        name=volume_name, persistent_volume_claim=claim
-                    )
+                    volume = client.V1Volume(name=volume_name, persistent_volume_claim=claim)
                     result.append(volume)
 
         # Handle host path mounts from service volumes
@@ -526,15 +601,11 @@ def volumes_for_pod_files(parsed_pod_files, spec, app_name):
                         mount_split = mount_string.split(":")
                         volume_source = mount_split[0]
                         if is_host_path_mount(volume_source):
-                            sanitized_name = sanitize_host_path_to_volume_name(
-                                volume_source
-                            )
+                            sanitized_name = sanitize_host_path_to_volume_name(volume_source)
                             if sanitized_name not in seen_host_path_volumes:
                                 seen_host_path_volumes.add(sanitized_name)
                                 # Create hostPath volume for mount inside kind node
-                                kind_mount_path = get_kind_host_path_mount_path(
-                                    sanitized_name
-                                )
+                                kind_mount_path = get_kind_host_path_mount_path(sanitized_name)
                                 host_path_source = client.V1HostPathVolumeSource(
                                     path=kind_mount_path, type="FileOrCreate"
                                 )
@@ -569,18 +640,18 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context):
     deployment_id = deployment_context.id
     backup_subdir = f"cluster-backups/{deployment_id}"
 
-    etcd_host_path = _make_absolute_host_path(
-        Path(f"./data/{backup_subdir}/etcd"), deployment_dir
-    )
+    etcd_host_path = _make_absolute_host_path(Path(f"./data/{backup_subdir}/etcd"), deployment_dir)
     volume_definitions.append(
-        f"  - hostPath: {etcd_host_path}\n" f"    containerPath: /var/lib/etcd\n"
+        f"  - hostPath: {etcd_host_path}\n"
+        f"    containerPath: /var/lib/etcd\n"
+        f"    propagation: HostToContainer\n"
     )
 
-    pki_host_path = _make_absolute_host_path(
-        Path(f"./data/{backup_subdir}/pki"), deployment_dir
-    )
+    pki_host_path = _make_absolute_host_path(Path(f"./data/{backup_subdir}/pki"), deployment_dir)
     volume_definitions.append(
-        f"  - hostPath: {pki_host_path}\n" f"    containerPath: /etc/kubernetes/pki\n"
+        f"  - hostPath: {pki_host_path}\n"
+        f"    containerPath: /etc/kubernetes/pki\n"
+        f"    propagation: HostToContainer\n"
     )
 
     # Note these paths are relative to the location of the pod files (at present)
@@ -606,21 +677,16 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context):
 
                         if is_host_path_mount(volume_name):
                             # Host path mount - add extraMount for kind
-                            sanitized_name = sanitize_host_path_to_volume_name(
-                                volume_name
-                            )
+                            sanitized_name = sanitize_host_path_to_volume_name(volume_name)
                             if sanitized_name not in seen_host_path_mounts:
                                 seen_host_path_mounts.add(sanitized_name)
                                 # Resolve path relative to compose directory
-                                host_path = resolve_host_path_for_kind(
-                                    volume_name, deployment_dir
-                                )
-                                container_path = get_kind_host_path_mount_path(
-                                    sanitized_name
-                                )
+                                host_path = resolve_host_path_for_kind(volume_name, deployment_dir)
+                                container_path = get_kind_host_path_mount_path(sanitized_name)
                                 volume_definitions.append(
                                     f"  - hostPath: {host_path}\n"
                                     f"    containerPath: {container_path}\n"
+                                    f"    propagation: HostToContainer\n"
                                 )
                                 if opts.o.debug:
                                     print(f"Added host path mount: {host_path}")
@@ -630,10 +696,7 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context):
                                 print(f"volume_name: {volume_name}")
                                 print(f"map: {volume_host_path_map}")
                                 print(f"mount path: {mount_path}")
-                            if (
-                                volume_name
-                                not in deployment_context.spec.get_configmaps()
-                            ):
+                            if volume_name not in deployment_context.spec.get_configmaps():
                                 if (
                                     volume_name in volume_host_path_map
                                     and volume_host_path_map[volume_name]
@@ -642,12 +705,11 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context):
                                         volume_host_path_map[volume_name],
                                         deployment_dir,
                                     )
-                                    container_path = get_kind_pv_bind_mount_path(
-                                        volume_name
-                                    )
+                                    container_path = get_kind_pv_bind_mount_path(volume_name)
                                     volume_definitions.append(
                                         f"  - hostPath: {host_path}\n"
                                         f"    containerPath: {container_path}\n"
+                                        f"    propagation: HostToContainer\n"
                                     )
     return (
         ""
@@ -671,8 +733,7 @@ def _generate_kind_port_mappings_from_services(parsed_pod_files):
                         # TODO handle the complex cases
                         # Looks like: 80 or something more complicated
                         port_definitions.append(
-                            f"  - containerPort: {port_string}\n"
-                            f"    hostPort: {port_string}\n"
+                            f"  - containerPort: {port_string}\n" f"    hostPort: {port_string}\n"
                         )
     return (
         ""
@@ -685,9 +746,7 @@ def _generate_kind_port_mappings(parsed_pod_files):
     port_definitions = []
     # Map port 80 and 443 for the Caddy ingress controller (HTTPS support)
     for port_string in ["80", "443"]:
-        port_definitions.append(
-            f"  - containerPort: {port_string}\n    hostPort: {port_string}\n"
-        )
+        port_definitions.append(f"  - containerPort: {port_string}\n    hostPort: {port_string}\n")
     return (
         ""
         if len(port_definitions) == 0
@@ -703,7 +762,11 @@ def _generate_high_memlock_spec_mount(deployment_dir: Path):
     references an absolute path.
     """
     spec_path = deployment_dir.joinpath(constants.high_memlock_spec_filename).resolve()
-    return f"  - hostPath: {spec_path}\n" f"    containerPath: {spec_path}\n"
+    return (
+        f"  - hostPath: {spec_path}\n"
+        f"    containerPath: {spec_path}\n"
+        f"    propagation: HostToContainer\n"
+    )
 
 
 def generate_high_memlock_spec_json():
@@ -877,27 +940,37 @@ def generate_cri_base_json():
     return generate_high_memlock_spec_json()
 
 
-def _generate_containerd_config_patches(
-    deployment_dir: Path, has_high_memlock: bool
-) -> str:
-    """Generate containerdConfigPatches YAML for custom runtime handlers.
+def _generate_containerd_config_patches(deployment_dir: Path, has_high_memlock: bool) -> str:
+    """Generate containerdConfigPatches YAML for containerd configuration.
 
-    This configures containerd to have a runtime handler named 'high-memlock'
-    that uses a custom OCI base spec with unlimited RLIMIT_MEMLOCK.
+    Includes:
+    - Local registry mirror (localhost:5001 -> http://kind-registry:5000)
+    - Custom runtime handler for high-memlock (if enabled)
     """
-    if not has_high_memlock:
+    patches = []
+
+    # Always configure the local registry mirror so kind nodes pull from it
+    registry_plugin = f'plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:{LOCAL_REGISTRY_HOST_PORT}"'
+    endpoint = f"http://{LOCAL_REGISTRY_NAME}:{LOCAL_REGISTRY_CONTAINER_PORT}"
+    patches.append(f"    [{registry_plugin}]\n" f'      endpoint = ["{endpoint}"]')
+
+    if has_high_memlock:
+        spec_path = deployment_dir.joinpath(constants.high_memlock_spec_filename).resolve()
+        runtime_name = constants.high_memlock_runtime
+        plugin_path = 'plugins."io.containerd.grpc.v1.cri".containerd.runtimes'
+        patches.append(
+            f"    [{plugin_path}.{runtime_name}]\n"
+            '      runtime_type = "io.containerd.runc.v2"\n'
+            f'      base_runtime_spec = "{spec_path}"'
+        )
+
+    if not patches:
         return ""
 
-    spec_path = deployment_dir.joinpath(constants.high_memlock_spec_filename).resolve()
-    runtime_name = constants.high_memlock_runtime
-    plugin_path = 'plugins."io.containerd.grpc.v1.cri".containerd.runtimes'
-    return (
-        "containerdConfigPatches:\n"
-        "  - |-\n"
-        f"    [{plugin_path}.{runtime_name}]\n"
-        '      runtime_type = "io.containerd.runc.v2"\n'
-        f'      base_runtime_spec = "{spec_path}"\n'
-    )
+    result = "containerdConfigPatches:\n"
+    for patch in patches:
+        result += "  - |-\n" + patch + "\n"
+    return result
 
 
 # Note: this makes any duplicate definition in b overwrite a
@@ -906,9 +979,7 @@ def merge_envs(a: Mapping[str, str], b: Mapping[str, str]) -> Mapping[str, str]:
     return result
 
 
-def _expand_shell_vars(
-    raw_val: str, env_map: Optional[Mapping[str, str]] = None
-) -> str:
+def _expand_shell_vars(raw_val: str, env_map: Mapping[str, str] | None = None) -> str:
     # Expand docker-compose style variable substitution:
     # ${VAR} - use VAR value or empty string
     # ${VAR:-default} - use VAR value or default if unset/empty
@@ -933,7 +1004,7 @@ def _expand_shell_vars(
 
 
 def envs_from_compose_file(
-    compose_file_envs: Mapping[str, str], env_map: Optional[Mapping[str, str]] = None
+    compose_file_envs: Mapping[str, str], env_map: Mapping[str, str] | None = None
 ) -> Mapping[str, str]:
     result = {}
     for env_var, env_val in compose_file_envs.items():
@@ -943,7 +1014,7 @@ def envs_from_compose_file(
 
 
 def translate_sidecar_service_names(
-    envs: Mapping[str, str], sibling_service_names: List[str]
+    envs: Mapping[str, str], sibling_service_names: list[str]
 ) -> Mapping[str, str]:
     """Translate docker-compose service names to localhost for sidecar containers.
 
@@ -970,7 +1041,12 @@ def translate_sidecar_service_names(
             # Handle URLs like: postgres://user:pass@db:5432/dbname
             # and simple refs like: db:5432 or just db
             pattern = rf"\b{re.escape(service_name)}(:\d+)?\b"
-            new_val = re.sub(pattern, lambda m: f'localhost{m.group(1) or ""}', new_val)
+
+            def _replace_with_localhost(m: re.Match[str]) -> str:
+                port: str = m.group(1) or ""
+                return "localhost" + port
+
+            new_val = re.sub(pattern, _replace_with_localhost, new_val)
 
         result[env_var] = new_val
 
@@ -978,8 +1054,8 @@ def translate_sidecar_service_names(
 
 
 def envs_from_environment_variables_map(
-    map: Mapping[str, str]
-) -> List[client.V1EnvVar]:
+    map: Mapping[str, str],
+) -> list[client.V1EnvVar]:
     result = []
     for env_var, env_val in map.items():
         result.append(client.V1EnvVar(env_var, env_val))
@@ -1010,17 +1086,13 @@ def generate_kind_config(deployment_dir: Path, deployment_context):
     pod_files = [p for p in compose_file_dir.iterdir() if p.is_file()]
     parsed_pod_files_map = parsed_pod_files_map_from_file_names(pod_files)
     port_mappings_yml = _generate_kind_port_mappings(parsed_pod_files_map)
-    mounts_yml = _generate_kind_mounts(
-        parsed_pod_files_map, deployment_dir, deployment_context
-    )
+    mounts_yml = _generate_kind_mounts(parsed_pod_files_map, deployment_dir, deployment_context)
 
     # Check if unlimited_memlock is enabled
     unlimited_memlock = deployment_context.spec.get_unlimited_memlock()
 
     # Generate containerdConfigPatches for RuntimeClass support
-    containerd_patches_yml = _generate_containerd_config_patches(
-        deployment_dir, unlimited_memlock
-    )
+    containerd_patches_yml = _generate_containerd_config_patches(deployment_dir, unlimited_memlock)
 
     # Add high-memlock spec file mount if needed
     if unlimited_memlock:

stack_orchestrator/deploy/spec.py

@@ -14,19 +14,18 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 import typing
-from typing import Optional
-import humanfriendly
-
 from pathlib import Path
 
-from stack_orchestrator.util import get_yaml
+import humanfriendly
+
 from stack_orchestrator import constants
+from stack_orchestrator.util import get_yaml
 
 
 class ResourceLimits:
-    cpus: Optional[float] = None
-    memory: Optional[int] = None
-    storage: Optional[int] = None
+    cpus: float | None = None
+    memory: int | None = None
+    storage: int | None = None
 
     def __init__(self, obj=None):
         if obj is None:
@@ -50,8 +49,8 @@ class ResourceLimits:
 
 
 class Resources:
-    limits: Optional[ResourceLimits] = None
-    reservations: Optional[ResourceLimits] = None
+    limits: ResourceLimits | None = None
+    reservations: ResourceLimits | None = None
 
     def __init__(self, obj=None):
         if obj is None:
@@ -74,9 +73,9 @@ class Resources:
 
 class Spec:
     obj: typing.Any
-    file_path: Optional[Path]
+    file_path: Path | None
 
-    def __init__(self, file_path: Optional[Path] = None, obj=None) -> None:
+    def __init__(self, file_path: Path | None = None, obj=None) -> None:
         if obj is None:
             obj = {}
         self.file_path = file_path
@@ -92,13 +91,13 @@ class Spec:
         return self.obj.get(item, default)
 
     def init_from_file(self, file_path: Path):
-        self.obj = get_yaml().load(open(file_path, "r"))
+        self.obj = get_yaml().load(open(file_path))
         self.file_path = file_path
 
     def get_image_registry(self):
         return self.obj.get(constants.image_registry_key)
 
-    def get_image_registry_config(self) -> typing.Optional[typing.Dict]:
+    def get_image_registry_config(self) -> dict | None:
         """Returns registry auth config: {server, username, token-env}.
 
         Used for private container registries like GHCR. The token-env field
@@ -107,7 +106,8 @@ class Spec:
         Note: Uses 'registry-credentials' key to avoid collision with
         'image-registry' key which is for pushing images.
         """
-        return self.obj.get("registry-credentials")
+        result: dict[str, str] | None = self.obj.get("registry-credentials")
+        return result
 
     def get_volumes(self):
         return self.obj.get(constants.volumes_key, {})
@@ -115,13 +115,23 @@ class Spec:
     def get_configmaps(self):
         return self.obj.get(constants.configmaps_key, {})
 
-    def get_secrets(self):
-        return self.obj.get(constants.secrets_key, {})
-
     def get_container_resources(self):
-        return Resources(
-            self.obj.get(constants.resources_key, {}).get("containers", {})
-        )
+        return Resources(self.obj.get(constants.resources_key, {}).get("containers", {}))
+
+    def get_container_resources_for(self, container_name: str) -> Resources | None:
+        """Look up per-container resource overrides from spec.yml.
+
+        Checks resources.containers.<container_name> in the spec. Returns None
+        if no per-container override exists (caller falls back to other sources).
+        """
+        containers_block = self.obj.get(constants.resources_key, {}).get("containers", {})
+        if container_name in containers_block:
+            entry = containers_block[container_name]
+            # Only treat it as a per-container override if it's a dict with
+            # reservations/limits nested inside (not a top-level global key)
+            if isinstance(entry, dict) and ("reservations" in entry or "limits" in entry):
+                return Resources(entry)
+        return None
 
     def get_container_resources_for(
         self, container_name: str
@@ -145,52 +155,11 @@ class Spec:
         return None
 
     def get_volume_resources(self):
-        return Resources(
-            self.obj.get(constants.resources_key, {}).get(constants.volumes_key, {})
-        )
-
-    def get_volume_resources_for(self, volume_name: str) -> typing.Optional[Resources]:
-        """Look up per-volume resource overrides from spec.yml.
-
-        Supports two formats under resources.volumes:
-
-        Global (original):
-            resources:
-              volumes:
-                reservations:
-                  storage: 5Gi
-
-        Per-volume (new):
-            resources:
-              volumes:
-                my-volume:
-                  reservations:
-                    storage: 10Gi
-
-        Returns the per-volume Resources if found, otherwise None.
-        The caller should fall back to get_volume_resources() then the default.
-        """
-        vol_section = (
-            self.obj.get(constants.resources_key, {}).get(constants.volumes_key, {})
-        )
-        if volume_name not in vol_section:
-            return None
-        entry = vol_section[volume_name]
-        if isinstance(entry, dict) and (
-            "reservations" in entry or "limits" in entry
-        ):
-            return Resources(entry)
-        return None
+        return Resources(self.obj.get(constants.resources_key, {}).get(constants.volumes_key, {}))
 
     def get_http_proxy(self):
         return self.obj.get(constants.network_key, {}).get(constants.http_proxy_key, [])
 
-    def get_namespace(self):
-        return self.obj.get("namespace")
-
-    def get_kind_cluster_name(self):
-        return self.obj.get("kind-cluster-name")
-
     def get_annotations(self):
         return self.obj.get(constants.annotations_key, {})
 
@@ -209,9 +178,7 @@ class Spec:
     def get_privileged(self):
         return (
             "true"
-            == str(
-                self.obj.get(constants.security_key, {}).get("privileged", "false")
-            ).lower()
+            == str(self.obj.get(constants.security_key, {}).get("privileged", "false")).lower()
         )
 
     def get_capabilities(self):
@@ -238,9 +205,7 @@ class Spec:
             Runtime class name string, or None to use default runtime.
         """
         # Explicit runtime class takes precedence
-        explicit = self.obj.get(constants.security_key, {}).get(
-            constants.runtime_class_key, None
-        )
+        explicit = self.obj.get(constants.security_key, {}).get(constants.runtime_class_key, None)
         if explicit:
             return explicit
 

stack_orchestrator/deploy/stack.py

@@ -13,8 +13,9 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from pathlib import Path
 import typing
+from pathlib import Path
+
 from stack_orchestrator.util import get_yaml
 
 
@@ -26,4 +27,4 @@ class Stack:
         self.name = name
 
     def init_from_file(self, file_path: Path):
-        self.obj = get_yaml().load(open(file_path, "r"))
+        self.obj = get_yaml().load(open(file_path))

stack_orchestrator/deploy/webapp/deploy_webapp.py

@@ -13,23 +13,22 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-import click
 import os
 from pathlib import Path
-from urllib.parse import urlparse
 from tempfile import NamedTemporaryFile
+from urllib.parse import urlparse
+
+import click
 
-from stack_orchestrator.util import error_exit, global_options2, get_yaml
-from stack_orchestrator.deploy.deployment_create import init_operation, create_operation
 from stack_orchestrator.deploy.deploy import create_deploy_context
 from stack_orchestrator.deploy.deploy_types import DeployCommandContext
+from stack_orchestrator.deploy.deployment_create import create_operation, init_operation
+from stack_orchestrator.util import error_exit, global_options2
 
 
 def _fixup_container_tag(deployment_dir: str, image: str):
     deployment_dir_path = Path(deployment_dir)
-    compose_file = deployment_dir_path.joinpath(
-        "compose", "docker-compose-webapp-template.yml"
-    )
+    compose_file = deployment_dir_path.joinpath("compose", "docker-compose-webapp-template.yml")
     # replace "cerc/webapp-container:local" in the file with our image tag
     with open(compose_file) as rfile:
         contents = rfile.read()
@@ -41,28 +40,22 @@ def _fixup_container_tag(deployment_dir: str, image: str):
 def _fixup_url_spec(spec_file_name: str, url: str):
     # url is like: https://example.com/path
     parsed_url = urlparse(url)
+    http_proxy_spec = f"""
+  http-proxy:
+    - host-name: {parsed_url.hostname}
+      routes:
+        - path: '{parsed_url.path if parsed_url.path else "/"}'
+          proxy-to: webapp:80
+    """
     spec_file_path = Path(spec_file_name)
-    yaml = get_yaml()
     with open(spec_file_path) as rfile:
-        contents = yaml.load(rfile)
-    contents.setdefault("network", {})["http-proxy"] = [
-        {
-            "host-name": parsed_url.hostname,
-            "routes": [
-                {
-                    "path": parsed_url.path if parsed_url.path else "/",
-                    "proxy-to": "webapp:80",
-                }
-            ],
-        }
-    ]
+        contents = rfile.read()
+        contents = contents + http_proxy_spec
     with open(spec_file_path, "w") as wfile:
-        yaml.dump(contents, wfile)
+        wfile.write(contents)
 
 
-def create_deployment(
-    ctx, deployment_dir, image, url, kube_config, image_registry, env_file
-):
+def create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
     # Do the equivalent of:
     # 1. laconic-so --stack webapp-template deploy --deploy-to k8s init \
     #    --output webapp-spec.yml
@@ -121,9 +114,7 @@ def command(ctx):
     "--image-registry",
     help="Provide a container image registry url for this k8s cluster",
 )
-@click.option(
-    "--deployment-dir", help="Create deployment files in this directory", required=True
-)
+@click.option("--deployment-dir", help="Create deployment files in this directory", required=True)
 @click.option("--image", help="image to deploy", required=True)
 @click.option("--url", help="url to serve", required=True)
 @click.option("--env-file", help="environment file for webapp")
@@ -131,6 +122,4 @@ def command(ctx):
 def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
     """create a deployment for the specified webapp container"""
 
-    return create_deployment(
-        ctx, deployment_dir, image, url, kube_config, image_registry, env_file
-    )
+    return create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file)

stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py

@@ -21,10 +21,10 @@ import sys
 import tempfile
 import time
 import uuid
-import yaml
 
 import click
 import gnupg
+import yaml
 
 from stack_orchestrator.deploy.images import remote_image_exists
 from stack_orchestrator.deploy.webapp import deploy_webapp
@@ -34,16 +34,16 @@ from stack_orchestrator.deploy.webapp.util import (
     TimedLogger,
     build_container_image,
     confirm_auction,
-    push_container_image,
-    file_hash,
-    deploy_to_k8s,
-    publish_deployment,
-    hostname_for_deployment_request,
-    generate_hostname_for_app,
-    match_owner,
-    skip_by_tag,
     confirm_payment,
+    deploy_to_k8s,
+    file_hash,
+    generate_hostname_for_app,
+    hostname_for_deployment_request,
     load_known_requests,
+    match_owner,
+    publish_deployment,
+    push_container_image,
+    skip_by_tag,
 )
 
 
@@ -70,9 +70,7 @@ def process_app_deployment_request(
     logger.log("BEGIN - process_app_deployment_request")
 
     # 1. look up application
-    app = laconic.get_record(
-        app_deployment_request.attributes.application, require=True
-    )
+    app = laconic.get_record(app_deployment_request.attributes.application, require=True)
     assert app is not None  # require=True ensures this
     logger.log(f"Retrieved app record {app_deployment_request.attributes.application}")
 
@@ -84,9 +82,7 @@ def process_app_deployment_request(
         if "allow" == fqdn_policy or "preexisting" == fqdn_policy:
             fqdn = requested_name
         else:
-            raise Exception(
-                f"{requested_name} is invalid: only unqualified hostnames are allowed."
-            )
+            raise Exception(f"{requested_name} is invalid: only unqualified hostnames are allowed.")
     else:
         fqdn = f"{requested_name}.{default_dns_suffix}"
 
@@ -108,8 +104,7 @@ def process_app_deployment_request(
             logger.log(f"Matched DnsRecord ownership: {matched_owner}")
         else:
             raise Exception(
-                "Unable to confirm ownership of DnsRecord %s for request %s"
-                % (dns_lrn, app_deployment_request.id)
+                f"Unable to confirm ownership of DnsRecord {dns_lrn} for request {app_deployment_request.id}"
             )
     elif "preexisting" == fqdn_policy:
         raise Exception(
@@ -144,7 +139,7 @@ def process_app_deployment_request(
         env_filename = tempfile.mktemp()
         with open(env_filename, "w") as file:
             for k, v in env.items():
-                file.write("%s=%s\n" % (k, shlex.quote(str(v))))
+                file.write(f"{k}={shlex.quote(str(v))}\n")
 
     # 5. determine new or existing deployment
     #   a. check for deployment lrn
@@ -153,8 +148,7 @@ def process_app_deployment_request(
         app_deployment_lrn = app_deployment_request.attributes.deployment
     if not app_deployment_lrn.startswith(deployment_record_namespace):
         raise Exception(
-            "Deployment LRN %s is not in a supported namespace"
-            % app_deployment_request.attributes.deployment
+            f"Deployment LRN {app_deployment_request.attributes.deployment} is not in a supported namespace"
         )
 
     deployment_record = laconic.get_record(app_deployment_lrn)
@@ -165,14 +159,14 @@ def process_app_deployment_request(
     # already-unique deployment id
     unique_deployment_id = hashlib.md5(fqdn.encode()).hexdigest()[:16]
     deployment_config_file = os.path.join(deployment_dir, "config.env")
-    deployment_container_tag = "laconic-webapp/%s:local" % unique_deployment_id
+    deployment_container_tag = f"laconic-webapp/{unique_deployment_id}:local"
     app_image_shared_tag = f"laconic-webapp/{app.id}:local"
     #   b. check for deployment directory (create if necessary)
     if not os.path.exists(deployment_dir):
         if deployment_record:
             raise Exception(
-                "Deployment record %s exists, but not deployment dir %s. "
-                "Please remove name." % (app_deployment_lrn, deployment_dir)
+                f"Deployment record {app_deployment_lrn} exists, but not deployment dir {deployment_dir}. "
+                "Please remove name."
             )
         logger.log(
             f"Creating webapp deployment in: {deployment_dir} "
@@ -198,11 +192,7 @@ def process_app_deployment_request(
         )
     # 6. build container (if needed)
     # TODO: add a comment that explains what this code is doing (not clear to me)
-    if (
-        not deployment_record
-        or deployment_record.attributes.application != app.id
-        or force_rebuild
-    ):
+    if not deployment_record or deployment_record.attributes.application != app.id or force_rebuild:
         needs_k8s_deploy = True
         # check if the image already exists
         shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag)
@@ -224,11 +214,9 @@ def process_app_deployment_request(
             # )
             logger.log("Tag complete")
         else:
-            extra_build_args = []  # TODO: pull from request
+            extra_build_args: list[str] = []  # TODO: pull from request
             logger.log(f"Building container image: {deployment_container_tag}")
-            build_container_image(
-                app, deployment_container_tag, extra_build_args, logger
-            )
+            build_container_image(app, deployment_container_tag, extra_build_args, logger)
             logger.log("Build complete")
             logger.log(f"Pushing container image: {deployment_container_tag}")
             push_container_image(deployment_dir, logger)
@@ -287,9 +275,7 @@ def dump_known_requests(filename, requests, status="SEEN"):
 
 @click.command()
 @click.option("--kube-config", help="Provide a config file for a k8s deployment")
-@click.option(
-    "--laconic-config", help="Provide a config file for laconicd", required=True
-)
+@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
 @click.option(
     "--image-registry",
     help="Provide a container image registry url for this k8s cluster",
@@ -306,9 +292,7 @@ def dump_known_requests(filename, requests, status="SEEN"):
     is_flag=True,
     default=False,
 )
-@click.option(
-    "--state-file", help="File to store state about previously seen requests."
-)
+@click.option("--state-file", help="File to store state about previously seen requests.")
 @click.option(
     "--only-update-state",
     help="Only update the state file, don't process any requests anything.",
@@ -331,9 +315,7 @@ def dump_known_requests(filename, requests, status="SEEN"):
     help="eg, lrn://laconic/deployments",
     required=True,
 )
-@click.option(
-    "--dry-run", help="Don't do anything, just report what would be done.", is_flag=True
-)
+@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
 @click.option(
     "--include-tags",
     help="Only include requests with matching tags (comma-separated).",
@@ -344,17 +326,13 @@ def dump_known_requests(filename, requests, status="SEEN"):
     help="Exclude requests with matching tags (comma-separated).",
     default="",
 )
-@click.option(
-    "--force-rebuild", help="Rebuild even if the image already exists.", is_flag=True
-)
+@click.option("--force-rebuild", help="Rebuild even if the image already exists.", is_flag=True)
 @click.option(
     "--recreate-on-deploy",
     help="Remove and recreate deployments instead of updating them.",
     is_flag=True,
 )
-@click.option(
-    "--log-dir", help="Output build/deployment logs to directory.", default=None
-)
+@click.option("--log-dir", help="Output build/deployment logs to directory.", default=None)
 @click.option(
     "--min-required-payment",
     help="Requests must have a minimum payment to be processed (in alnt)",
@@ -378,9 +356,7 @@ def dump_known_requests(filename, requests, status="SEEN"):
     help="The directory containing uploaded config.",
     required=True,
 )
-@click.option(
-    "--private-key-file", help="The private key for decrypting config.", required=True
-)
+@click.option("--private-key-file", help="The private key for decrypting config.", required=True)
 @click.option(
     "--registry-lock-file",
     help="File path to use for registry mutex lock",
@@ -435,11 +411,7 @@ def command(  # noqa: C901
         sys.exit(2)
 
     if not only_update_state:
-        if (
-            not record_namespace_dns
-            or not record_namespace_deployments
-            or not dns_suffix
-        ):
+        if not record_namespace_dns or not record_namespace_deployments or not dns_suffix:
             print(
                 "--dns-suffix, --record-namespace-dns, and "
                 "--record-namespace-deployments are all required",
@@ -491,8 +463,7 @@ def command(  # noqa: C901
 
         if min_required_payment and not payment_address:
             print(
-                f"Minimum payment required, but no payment address listed "
-                f"for deployer: {lrn}.",
+                f"Minimum payment required, but no payment address listed " f"for deployer: {lrn}.",
                 file=sys.stderr,
             )
             sys.exit(2)
@@ -557,26 +528,18 @@ def command(  # noqa: C901
                 requested_name = r.attributes.dns
                 if not requested_name:
                     requested_name = generate_hostname_for_app(app)
-                    main_logger.log(
-                        "Generating name %s for request %s." % (requested_name, r_id)
-                    )
+                    main_logger.log(f"Generating name {requested_name} for request {r_id}.")
 
-                if (
-                    requested_name in skipped_by_name
-                    or requested_name in requests_by_name
-                ):
-                    main_logger.log(
-                        "Ignoring request %s, it has been superseded." % r_id
-                    )
+                if requested_name in skipped_by_name or requested_name in requests_by_name:
+                    main_logger.log(f"Ignoring request {r_id}, it has been superseded.")
                     result = "SKIP"
                     continue
 
                 if skip_by_tag(r, include_tags, exclude_tags):
                     r_tags = r.attributes.tags if r.attributes else None
                     main_logger.log(
-                        "Skipping request %s, filtered by tag "
-                        "(include %s, exclude %s, present %s)"
-                        % (r_id, include_tags, exclude_tags, r_tags)
+                        f"Skipping request {r_id}, filtered by tag "
+                        f"(include {include_tags}, exclude {exclude_tags}, present {r_tags})"
                     )
                     skipped_by_name[requested_name] = r
                     result = "SKIP"
@@ -584,8 +547,7 @@ def command(  # noqa: C901
 
                 r_app = r.attributes.application if r.attributes else "unknown"
                 main_logger.log(
-                    "Found pending request %s to run application %s on %s."
-                    % (r_id, r_app, requested_name)
+                    f"Found pending request {r_id} to run application {r_app} on {requested_name}."
                 )
                 requests_by_name[requested_name] = r
             except Exception as e:
@@ -617,17 +579,14 @@ def command(  # noqa: C901
 
         requests_to_check_for_payment = []
         for r in requests_by_name.values():
-            if r.id in cancellation_requests and match_owner(
-                cancellation_requests[r.id], r
-            ):
+            if r.id in cancellation_requests and match_owner(cancellation_requests[r.id], r):
                 main_logger.log(
                     f"Found deployment cancellation request for {r.id} "
                     f"at {cancellation_requests[r.id].id}"
                 )
             elif r.id in deployments_by_request:
                 main_logger.log(
-                    f"Found satisfied request for {r.id} "
-                    f"at {deployments_by_request[r.id].id}"
+                    f"Found satisfied request for {r.id} " f"at {deployments_by_request[r.id].id}"
                 )
             else:
                 if (
@@ -635,8 +594,7 @@ def command(  # noqa: C901
                     and previous_requests[r.id].get("status", "") != "RETRY"
                 ):
                     main_logger.log(
-                        f"Skipping unsatisfied request {r.id} "
-                        "because we have seen it before."
+                        f"Skipping unsatisfied request {r.id} " "because we have seen it before."
                     )
                 else:
                     main_logger.log(f"Request {r.id} needs to processed.")
@@ -650,14 +608,10 @@ def command(  # noqa: C901
                         main_logger.log(f"{r.id}: Auction confirmed.")
                         requests_to_execute.append(r)
                     else:
-                        main_logger.log(
-                            f"Skipping request {r.id}: unable to verify auction."
-                        )
+                        main_logger.log(f"Skipping request {r.id}: unable to verify auction.")
                         dump_known_requests(state_file, [r], status="SKIP")
                 else:
-                    main_logger.log(
-                        f"Skipping request {r.id}: not handling requests with auction."
-                    )
+                    main_logger.log(f"Skipping request {r.id}: not handling requests with auction.")
                     dump_known_requests(state_file, [r], status="SKIP")
             elif min_required_payment:
                 main_logger.log(f"{r.id}: Confirming payment...")
@@ -671,16 +625,12 @@ def command(  # noqa: C901
                     main_logger.log(f"{r.id}: Payment confirmed.")
                     requests_to_execute.append(r)
                 else:
-                    main_logger.log(
-                        f"Skipping request {r.id}: unable to verify payment."
-                    )
+                    main_logger.log(f"Skipping request {r.id}: unable to verify payment.")
                     dump_known_requests(state_file, [r], status="UNPAID")
             else:
                 requests_to_execute.append(r)
 
-        main_logger.log(
-            "Found %d unsatisfied request(s) to process." % len(requests_to_execute)
-        )
+        main_logger.log(f"Found {len(requests_to_execute)} unsatisfied request(s) to process.")
 
         if not dry_run:
             for r in requests_to_execute:
@@ -700,10 +650,8 @@ def command(  # noqa: C901
                         if not os.path.exists(run_log_dir):
                             os.mkdir(run_log_dir)
                         run_log_file_path = os.path.join(run_log_dir, f"{run_id}.log")
-                        main_logger.log(
-                            f"Directing deployment logs to: {run_log_file_path}"
-                        )
-                        run_log_file = open(run_log_file_path, "wt")
+                        main_logger.log(f"Directing deployment logs to: {run_log_file_path}")
+                        run_log_file = open(run_log_file_path, "w")
                         run_reg_client = LaconicRegistryClient(
                             laconic_config,
                             log_file=run_log_file,

stack_orchestrator/deploy/webapp/handle_deployment_auction.py

@@ -12,18 +12,18 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-import sys
 import json
+import sys
 
 import click
 
 from stack_orchestrator.deploy.webapp.util import (
+    AUCTION_KIND_PROVIDER,
     AttrDict,
+    AuctionStatus,
     LaconicRegistryClient,
     TimedLogger,
     load_known_requests,
-    AUCTION_KIND_PROVIDER,
-    AuctionStatus,
 )
 
 
@@ -44,16 +44,13 @@ def process_app_deployment_auction(
 
     # Check auction kind
     if auction.kind != AUCTION_KIND_PROVIDER:
-        raise Exception(
-            f"Auction kind needs to be ${AUCTION_KIND_PROVIDER}, got {auction.kind}"
-        )
+        raise Exception(f"Auction kind needs to be ${AUCTION_KIND_PROVIDER}, got {auction.kind}")
 
     if current_status == "PENDING":
         # Skip if pending auction not in commit state
         if auction.status != AuctionStatus.COMMIT:
             logger.log(
-                f"Skipping pending request, auction {auction_id} "
-                f"status: {auction.status}"
+                f"Skipping pending request, auction {auction_id} " f"status: {auction.status}"
             )
             return "SKIP", ""
 
@@ -115,9 +112,7 @@ def dump_known_auction_requests(filename, requests, status="SEEN"):
 
 
 @click.command()
-@click.option(
-    "--laconic-config", help="Provide a config file for laconicd", required=True
-)
+@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
 @click.option(
     "--state-file",
     help="File to store state about previously seen auction requests.",
@@ -133,9 +128,7 @@ def dump_known_auction_requests(filename, requests, status="SEEN"):
     help="File path to use for registry mutex lock",
     default=None,
 )
-@click.option(
-    "--dry-run", help="Don't do anything, just report what would be done.", is_flag=True
-)
+@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
 @click.pass_context
 def command(
     ctx,
@@ -198,8 +191,7 @@ def command(
                         continue
 
                     logger.log(
-                        f"Found pending auction request {r.id} for application "
-                        f"{application}."
+                        f"Found pending auction request {r.id} for application " f"{application}."
                     )
 
                 # Add requests to be processed
@@ -209,9 +201,7 @@ def command(
                 result_status = "ERROR"
                 logger.log(f"ERROR: examining request {r.id}: " + str(e))
             finally:
-                logger.log(
-                    f"DONE: Examining request {r.id} with result {result_status}."
-                )
+                logger.log(f"DONE: Examining request {r.id} with result {result_status}.")
                 if result_status in ["ERROR"]:
                     dump_known_auction_requests(
                         state_file,

stack_orchestrator/deploy/webapp/publish_deployment_auction.py

@@ -30,9 +30,7 @@ def fatal(msg: str):
 
 
 @click.command()
-@click.option(
-    "--laconic-config", help="Provide a config file for laconicd", required=True
-)
+@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
 @click.option(
     "--app",
     help="The LRN of the application to deploy.",

stack_orchestrator/deploy/webapp/publish_webapp_deployer.py

@@ -13,28 +13,24 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 import base64
-import click
 import sys
-import yaml
-
 from urllib.parse import urlparse
 
+import click
+import yaml
+
 from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient
 
 
 @click.command()
-@click.option(
-    "--laconic-config", help="Provide a config file for laconicd", required=True
-)
+@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
 @click.option("--api-url", help="The API URL of the deployer.", required=True)
 @click.option(
     "--public-key-file",
     help="The public key to use.  This should be a binary file.",
     required=True,
 )
-@click.option(
-    "--lrn", help="eg, lrn://laconic/deployers/my.deployer.name", required=True
-)
+@click.option("--lrn", help="eg, lrn://laconic/deployers/my.deployer.name", required=True)
 @click.option(
     "--payment-address",
     help="The address to which payments should be made.  "
@@ -84,9 +80,7 @@ def command(  # noqa: C901
     }
 
     if min_required_payment:
-        webapp_deployer_record["record"][
-            "minimumPayment"
-        ] = f"{min_required_payment}alnt"
+        webapp_deployer_record["record"]["minimumPayment"] = f"{min_required_payment}alnt"
 
     if dry_run:
         yaml.dump(webapp_deployer_record, sys.stdout)

stack_orchestrator/deploy/webapp/registry_mutex.py

@@ -1,6 +1,6 @@
-from functools import wraps
 import os
 import time
+from functools import wraps
 
 # Define default file path for the lock
 DEFAULT_LOCK_FILE_PATH = "/tmp/registry_mutex_lock_file"
@@ -17,7 +17,7 @@ def acquire_lock(client, lock_file_path, timeout):
         try:
             # Check if lock file exists and is potentially stale
             if os.path.exists(lock_file_path):
-                with open(lock_file_path, "r") as lock_file:
+                with open(lock_file_path) as lock_file:
                     timestamp = float(lock_file.read().strip())
 
                 # If lock is stale, remove the lock file
@@ -25,9 +25,7 @@ def acquire_lock(client, lock_file_path, timeout):
                     print(f"Stale lock detected, removing lock file {lock_file_path}")
                     os.remove(lock_file_path)
                 else:
-                    print(
-                        f"Lock file {lock_file_path} exists and is recent, waiting..."
-                    )
+                    print(f"Lock file {lock_file_path} exists and is recent, waiting...")
                     time.sleep(LOCK_RETRY_INTERVAL)
                     continue
 

stack_orchestrator/deploy/webapp/request_webapp_deployment.py

@@ -12,24 +12,24 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
+import base64
 import shutil
 import sys
 import tempfile
 from datetime import datetime
 from typing import NoReturn
-import base64
 
-import gnupg
 import click
+import gnupg
 import requests
 import yaml
+from dotenv import dotenv_values
 
 from stack_orchestrator.deploy.webapp.util import (
     AUCTION_KIND_PROVIDER,
     AuctionStatus,
     LaconicRegistryClient,
 )
-from dotenv import dotenv_values
 
 
 def fatal(msg: str) -> NoReturn:
@@ -38,9 +38,7 @@ def fatal(msg: str) -> NoReturn:
 
 
 @click.command()
-@click.option(
-    "--laconic-config", help="Provide a config file for laconicd", required=True
-)
+@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
 @click.option(
     "--app",
     help="The LRN of the application to deploy.",
@@ -63,9 +61,7 @@ def fatal(msg: str) -> NoReturn:
         "'auto' to use the deployer's minimum required payment."
     ),
 )
-@click.option(
-    "--use-payment", help="The TX id of an existing, unused payment", default=None
-)
+@click.option("--use-payment", help="The TX id of an existing, unused payment", default=None)
 @click.option("--dns", help="the DNS name to request (default is autogenerated)")
 @click.option(
     "--dry-run",
@@ -144,9 +140,7 @@ def command(  # noqa: C901
         # Check auction kind
         auction_kind = auction.kind if auction else None
         if auction_kind != AUCTION_KIND_PROVIDER:
-            fatal(
-                f"Auction kind needs to be ${AUCTION_KIND_PROVIDER}, got {auction_kind}"
-            )
+            fatal(f"Auction kind needs to be ${AUCTION_KIND_PROVIDER}, got {auction_kind}")
 
         # Check auction status
         auction_status = auction.status if auction else None
@@ -163,14 +157,9 @@ def command(  # noqa: C901
         # Get deployer record for all the auction winners
         for auction_winner in auction_winners:
             # TODO: Match auction winner address with provider address?
-            deployer_records_by_owner = laconic.webapp_deployers(
-                {"paymentAddress": auction_winner}
-            )
+            deployer_records_by_owner = laconic.webapp_deployers({"paymentAddress": auction_winner})
             if len(deployer_records_by_owner) == 0:
-                print(
-                    f"WARNING: Unable to locate deployer for auction winner "
-                    f"{auction_winner}"
-                )
+                print(f"WARNING: Unable to locate deployer for auction winner " f"{auction_winner}")
 
             # Take first record with name set
             target_deployer_record = deployer_records_by_owner[0]
@@ -196,9 +185,7 @@ def command(  # noqa: C901
                 gpg = gnupg.GPG(gnupghome=tempdir)
 
                 # Import the deployer's public key
-                result = gpg.import_keys(
-                    base64.b64decode(deployer_record.attributes.publicKey)
-                )
+                result = gpg.import_keys(base64.b64decode(deployer_record.attributes.publicKey))
                 if 1 != result.imported:
                     fatal("Failed to import deployer's public key.")
 
@@ -237,15 +224,9 @@ def command(  # noqa: C901
         if (not deployer) and len(deployer_record.names):
             target_deployer = deployer_record.names[0]
 
-        app_name = (
-            app_record.attributes.name
-            if app_record and app_record.attributes
-            else "unknown"
-        )
+        app_name = app_record.attributes.name if app_record and app_record.attributes else "unknown"
         app_version = (
-            app_record.attributes.version
-            if app_record and app_record.attributes
-            else "unknown"
+            app_record.attributes.version if app_record and app_record.attributes else "unknown"
         )
         deployment_request = {
             "record": {
@@ -273,15 +254,11 @@ def command(  # noqa: C901
                 deployment_request["record"]["payment"] = "DRY_RUN"
             elif "auto" == make_payment:
                 if "minimumPayment" in deployer_record.attributes:
-                    amount = int(
-                        deployer_record.attributes.minimumPayment.replace("alnt", "")
-                    )
+                    amount = int(deployer_record.attributes.minimumPayment.replace("alnt", ""))
             else:
                 amount = make_payment
             if amount:
-                receipt = laconic.send_tokens(
-                    deployer_record.attributes.paymentAddress, amount
-                )
+                receipt = laconic.send_tokens(deployer_record.attributes.paymentAddress, amount)
                 deployment_request["record"]["payment"] = receipt.tx.hash
                 print("Payment TX:", receipt.tx.hash)
         elif use_payment:

stack_orchestrator/deploy/webapp/request_webapp_undeployment.py

@@ -26,12 +26,8 @@ def fatal(msg: str) -> None:
 
 
 @click.command()
-@click.option(
-    "--laconic-config", help="Provide a config file for laconicd", required=True
-)
-@click.option(
-    "--deployer", help="The LRN of the deployer to process this request.", required=True
-)
+@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
+@click.option("--deployer", help="The LRN of the deployer to process this request.", required=True)
 @click.option(
     "--deployment",
     help="Deployment record (ApplicationDeploymentRecord) id of the deployment.",
@@ -44,9 +40,7 @@ def fatal(msg: str) -> None:
         "'auto' to use the deployer's minimum required payment."
     ),
 )
-@click.option(
-    "--use-payment", help="The TX id of an existing, unused payment", default=None
-)
+@click.option("--use-payment", help="The TX id of an existing, unused payment", default=None)
 @click.option(
     "--dry-run",
     help="Don't publish anything, just report what would be done.",

stack_orchestrator/deploy/webapp/run_webapp.py

@@ -22,6 +22,7 @@
 # all or specific containers
 
 import hashlib
+
 import click
 from dotenv import dotenv_values
 

stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py

@@ -21,11 +21,11 @@ import sys
 import click
 
 from stack_orchestrator.deploy.webapp.util import (
-    TimedLogger,
     LaconicRegistryClient,
+    TimedLogger,
+    confirm_payment,
     match_owner,
     skip_by_tag,
-    confirm_payment,
 )
 
 main_logger = TimedLogger(file=sys.stderr)
@@ -40,9 +40,7 @@ def process_app_removal_request(
     delete_names,
     webapp_deployer_record,
 ):
-    deployment_record = laconic.get_record(
-        app_removal_request.attributes.deployment, require=True
-    )
+    deployment_record = laconic.get_record(app_removal_request.attributes.deployment, require=True)
     assert deployment_record is not None  # require=True ensures this
     assert deployment_record.attributes is not None
 
@@ -50,12 +48,10 @@ def process_app_removal_request(
     assert dns_record is not None  # require=True ensures this
     assert dns_record.attributes is not None
 
-    deployment_dir = os.path.join(
-        deployment_parent_dir, dns_record.attributes.name.lower()
-    )
+    deployment_dir = os.path.join(deployment_parent_dir, dns_record.attributes.name.lower())
 
     if not os.path.exists(deployment_dir):
-        raise Exception("Deployment directory %s does not exist." % deployment_dir)
+        raise Exception(f"Deployment directory {deployment_dir} does not exist.")
 
     # Check if the removal request is from the owner of the DnsRecord or
     # deployment record.
@@ -63,9 +59,7 @@ def process_app_removal_request(
 
     # Or of the original deployment request.
     if not matched_owner and deployment_record.attributes.request:
-        original_request = laconic.get_record(
-            deployment_record.attributes.request, require=True
-        )
+        original_request = laconic.get_record(deployment_record.attributes.request, require=True)
         assert original_request is not None  # require=True ensures this
         matched_owner = match_owner(app_removal_request, original_request)
 
@@ -75,8 +69,7 @@ def process_app_removal_request(
         deployment_id = deployment_record.id if deployment_record else "unknown"
         request_id = app_removal_request.id if app_removal_request else "unknown"
         raise Exception(
-            "Unable to confirm ownership of deployment %s for removal request %s"
-            % (deployment_id, request_id)
+            f"Unable to confirm ownership of deployment {deployment_id} for removal request {request_id}"
         )
 
     # TODO(telackey): Call the function directly. The easiest way to build
@@ -124,7 +117,7 @@ def process_app_removal_request(
 
 def load_known_requests(filename):
     if filename and os.path.exists(filename):
-        return json.load(open(filename, "r"))
+        return json.load(open(filename))
     return {}
 
 
@@ -138,9 +131,7 @@ def dump_known_requests(filename, requests):
 
 
 @click.command()
-@click.option(
-    "--laconic-config", help="Provide a config file for laconicd", required=True
-)
+@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
 @click.option(
     "--deployment-parent-dir",
     help="Create deployment directories beneath this directory",
@@ -153,9 +144,7 @@ def dump_known_requests(filename, requests):
     is_flag=True,
     default=False,
 )
-@click.option(
-    "--state-file", help="File to store state about previously seen requests."
-)
+@click.option("--state-file", help="File to store state about previously seen requests.")
 @click.option(
     "--only-update-state",
     help="Only update the state file, don't process any requests anything.",
@@ -166,12 +155,8 @@ def dump_known_requests(filename, requests):
     help="Delete all names associated with removed deployments.",
     default=True,
 )
-@click.option(
-    "--delete-volumes/--preserve-volumes", default=True, help="delete data volumes"
-)
-@click.option(
-    "--dry-run", help="Don't do anything, just report what would be done.", is_flag=True
-)
+@click.option("--delete-volumes/--preserve-volumes", default=True, help="delete data volumes")
+@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
 @click.option(
     "--include-tags",
     help="Only include requests with matching tags (comma-separated).",
@@ -245,8 +230,7 @@ def command(  # noqa: C901
 
     if min_required_payment and not payment_address:
         print(
-            f"Minimum payment required, but no payment address listed "
-            f"for deployer: {lrn}.",
+            f"Minimum payment required, but no payment address listed " f"for deployer: {lrn}.",
             file=sys.stderr,
         )
         sys.exit(2)
@@ -303,9 +287,7 @@ def command(  # noqa: C901
             continue
         if not r.attributes.deployment:
             r_id = r.id if r else "unknown"
-            main_logger.log(
-                f"Skipping removal request {r_id} since it was a cancellation."
-            )
+            main_logger.log(f"Skipping removal request {r_id} since it was a cancellation.")
         elif r.attributes.deployment in one_per_deployment:
             r_id = r.id if r else "unknown"
             main_logger.log(f"Skipping removal request {r_id} since it was superseded.")
@@ -323,14 +305,12 @@ def command(  # noqa: C901
                 )
             elif skip_by_tag(r, include_tags, exclude_tags):
                 main_logger.log(
-                    "Skipping removal request %s, filtered by tag "
-                    "(include %s, exclude %s, present %s)"
-                    % (r.id, include_tags, exclude_tags, r.attributes.tags)
+                    f"Skipping removal request {r.id}, filtered by tag "
+                    f"(include {include_tags}, exclude {exclude_tags}, present {r.attributes.tags})"
                 )
             elif r.id in removals_by_request:
                 main_logger.log(
-                    f"Found satisfied request for {r.id} "
-                    f"at {removals_by_request[r.id].id}"
+                    f"Found satisfied request for {r.id} " f"at {removals_by_request[r.id].id}"
                 )
             elif r.attributes.deployment in removals_by_deployment:
                 main_logger.log(
@@ -344,8 +324,7 @@ def command(  # noqa: C901
                     requests_to_check_for_payment.append(r)
                 else:
                     main_logger.log(
-                        f"Skipping unsatisfied request {r.id} "
-                        "because we have seen it before."
+                        f"Skipping unsatisfied request {r.id} " "because we have seen it before."
                     )
         except Exception as e:
             main_logger.log(f"ERROR examining {r.id}: {e}")
@@ -370,9 +349,7 @@ def command(  # noqa: C901
     else:
         requests_to_execute = requests_to_check_for_payment
 
-    main_logger.log(
-        "Found %d unsatisfied request(s) to process." % len(requests_to_execute)
-    )
+    main_logger.log(f"Found {len(requests_to_execute)} unsatisfied request(s) to process.")
 
     if not dry_run:
         for r in requests_to_execute:

stack_orchestrator/deploy/webapp/util.py

@@ -22,10 +22,10 @@ import subprocess
 import sys
 import tempfile
 import uuid
-import yaml
-
 from enum import Enum
-from typing import Any, List, Optional, TextIO
+from typing import Any, TextIO
+
+import yaml
 
 from stack_orchestrator.deploy.webapp.registry_mutex import registry_mutex
 
@@ -43,17 +43,17 @@ AUCTION_KIND_PROVIDER = "provider"
 
 class AttrDict(dict):
     def __init__(self, *args: Any, **kwargs: Any) -> None:
-        super(AttrDict, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.__dict__ = self
 
     def __getattribute__(self, attr: str) -> Any:
-        __dict__ = super(AttrDict, self).__getattribute__("__dict__")
+        __dict__ = super().__getattribute__("__dict__")
         if attr in __dict__:
-            v = super(AttrDict, self).__getattribute__(attr)
+            v = super().__getattribute__(attr)
             if isinstance(v, dict):
                 return AttrDict(v)
             return v
-        return super(AttrDict, self).__getattribute__(attr)
+        return super().__getattribute__(attr)
 
     def __getattr__(self, attr: str) -> Any:
         # This method is called when attribute is not found
@@ -62,15 +62,13 @@ class AttrDict(dict):
 
 
 class TimedLogger:
-    def __init__(self, id: str = "", file: Optional[TextIO] = None) -> None:
+    def __init__(self, id: str = "", file: TextIO | None = None) -> None:
         self.start = datetime.datetime.now()
         self.last = self.start
         self.id = id
         self.file = file
 
-    def log(
-        self, msg: str, show_step_time: bool = True, show_total_time: bool = False
-    ) -> None:
+    def log(self, msg: str, show_step_time: bool = True, show_total_time: bool = False) -> None:
         prefix = f"{datetime.datetime.utcnow()} - {self.id}"
         if show_step_time:
             prefix += f" - {datetime.datetime.now() - self.last} (step)"
@@ -84,11 +82,11 @@ class TimedLogger:
 
 def load_known_requests(filename):
     if filename and os.path.exists(filename):
-        return json.load(open(filename, "r"))
+        return json.load(open(filename))
     return {}
 
 
-def logged_cmd(log_file: Optional[TextIO], *vargs: str) -> str:
+def logged_cmd(log_file: TextIO | None, *vargs: str) -> str:
     result = None
     try:
         if log_file:
@@ -105,15 +103,14 @@ def logged_cmd(log_file: Optional[TextIO], *vargs: str) -> str:
         raise err
 
 
-def match_owner(
-    recordA: Optional[AttrDict], *records: Optional[AttrDict]
-) -> Optional[str]:
+def match_owner(recordA: AttrDict | None, *records: AttrDict | None) -> str | None:
     if not recordA or not recordA.owners:
         return None
     for owner in recordA.owners:
         for otherRecord in records:
             if otherRecord and otherRecord.owners and owner in otherRecord.owners:
-                return owner
+                result: str | None = owner
+                return result
     return None
 
 
@@ -147,9 +144,7 @@ class LaconicRegistryClient:
             return self.cache["whoami"]
 
         args = ["laconic", "-c", self.config_file, "registry", "account", "get"]
-        results = [
-            AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
-        ]
+        results = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r]
 
         if len(results):
             self.cache["whoami"] = results[0]
@@ -178,9 +173,7 @@ class LaconicRegistryClient:
             "--address",
             address,
         ]
-        results = [
-            AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
-        ]
+        results = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r]
         if len(results):
             self.cache["accounts"][address] = results[0]
             return results[0]
@@ -203,9 +196,7 @@ class LaconicRegistryClient:
             "--id",
             id,
         ]
-        results = [
-            AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
-        ]
+        results = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r]
         self._add_to_cache(results)
         if len(results):
             return results[0]
@@ -216,9 +207,7 @@ class LaconicRegistryClient:
 
     def list_bonds(self):
         args = ["laconic", "-c", self.config_file, "registry", "bond", "list"]
-        results = [
-            AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
-        ]
+        results = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r]
         self._add_to_cache(results)
         return results
 
@@ -232,12 +221,10 @@ class LaconicRegistryClient:
 
         if criteria:
             for k, v in criteria.items():
-                args.append("--%s" % k)
+                args.append(f"--{k}")
                 args.append(str(v))
 
-        results = [
-            AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
-        ]
+        results = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r]
 
         # Most recent records first
         results.sort(key=lambda r: r.createTime or "")
@@ -246,7 +233,7 @@ class LaconicRegistryClient:
 
         return results
 
-    def _add_to_cache(self, records: List[AttrDict]) -> None:
+    def _add_to_cache(self, records: list[AttrDict]) -> None:
         if not records:
             return
 
@@ -271,9 +258,7 @@ class LaconicRegistryClient:
 
         args = ["laconic", "-c", self.config_file, "registry", "name", "resolve", name]
 
-        parsed = [
-            AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
-        ]
+        parsed = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r]
         if parsed:
             self._add_to_cache(parsed)
             return parsed[0]
@@ -303,9 +288,7 @@ class LaconicRegistryClient:
             name_or_id,
         ]
 
-        parsed = [
-            AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
-        ]
+        parsed = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r]
         if len(parsed):
             self._add_to_cache(parsed)
             return parsed[0]
@@ -356,9 +339,7 @@ class LaconicRegistryClient:
 
         results = None
         try:
-            results = [
-                AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r
-            ]
+            results = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args)) if r]
         except:  # noqa: E722
             pass
 
@@ -422,7 +403,7 @@ class LaconicRegistryClient:
             record_file = open(record_fname, "w")
             yaml.dump(record, record_file)
             record_file.close()
-            print(open(record_fname, "r").read(), file=self.log_file)
+            print(open(record_fname).read(), file=self.log_file)
 
             new_record_id = json.loads(
                 logged_cmd(
@@ -573,10 +554,10 @@ def determine_base_container(clone_dir, app_type="webapp"):
 
 
 def build_container_image(
-    app_record: Optional[AttrDict],
+    app_record: AttrDict | None,
     tag: str,
-    extra_build_args: Optional[List[str]] = None,
-    logger: Optional[TimedLogger] = None,
+    extra_build_args: list[str] | None = None,
+    logger: TimedLogger | None = None,
 ) -> None:
     if app_record is None:
         raise ValueError("app_record cannot be None")
@@ -649,9 +630,7 @@ def build_container_image(
             )
             result.check_returncode()
 
-        base_container = determine_base_container(
-            clone_dir, app_record.attributes.app_type
-        )
+        base_container = determine_base_container(clone_dir, app_record.attributes.app_type)
 
         if logger:
             logger.log("Building webapp ...")
@@ -696,7 +675,7 @@ def deploy_to_k8s(deploy_record, deployment_dir, recreate, logger):
         if not deploy_record:
             commands_to_run = ["start"]
         else:
-            commands_to_run = ["update"]
+            commands_to_run = ["update-envs"]
 
     for command in commands_to_run:
         logger.log(f"Running {command} command on deployment dir: {deployment_dir}")
@@ -727,14 +706,12 @@ def publish_deployment(
     if not deploy_record:
         deploy_ver = "0.0.1"
     else:
-        deploy_ver = "0.0.%d" % (
-            int(deploy_record.attributes.version.split(".")[-1]) + 1
-        )
+        deploy_ver = f"0.0.{int(deploy_record.attributes.version.split('.')[-1]) + 1}"
 
     if not dns_record:
         dns_ver = "0.0.1"
     else:
-        dns_ver = "0.0.%d" % (int(dns_record.attributes.version.split(".")[-1]) + 1)
+        dns_ver = f"0.0.{int(dns_record.attributes.version.split('.')[-1]) + 1}"
 
     spec = yaml.full_load(open(os.path.join(deployment_dir, "spec.yml")))
     fqdn = spec["network"]["http-proxy"][0]["host-name"]
@@ -779,13 +756,9 @@ def publish_deployment(
 
         # Set auction or payment id from request
         if app_deployment_request.attributes.auction:
-            new_deployment_record["record"][
-                "auction"
-            ] = app_deployment_request.attributes.auction
+            new_deployment_record["record"]["auction"] = app_deployment_request.attributes.auction
         elif app_deployment_request.attributes.payment:
-            new_deployment_record["record"][
-                "payment"
-            ] = app_deployment_request.attributes.payment
+            new_deployment_record["record"]["payment"] = app_deployment_request.attributes.payment
 
     if webapp_deployer_record:
         new_deployment_record["record"]["deployer"] = webapp_deployer_record.names[0]
@@ -799,9 +772,7 @@ def publish_deployment(
 def hostname_for_deployment_request(app_deployment_request, laconic):
     dns_name = app_deployment_request.attributes.dns
     if not dns_name:
-        app = laconic.get_record(
-            app_deployment_request.attributes.application, require=True
-        )
+        app = laconic.get_record(app_deployment_request.attributes.application, require=True)
         dns_name = generate_hostname_for_app(app)
     elif dns_name.startswith("lrn://"):
         record = laconic.get_record(dns_name, require=True)
@@ -818,7 +789,7 @@ def generate_hostname_for_app(app):
         m.update(app.attributes.repository[0].encode())
     else:
         m.update(app.attributes.repository.encode())
-    return "%s-%s" % (last_part, m.hexdigest()[0:10])
+    return f"{last_part}-{m.hexdigest()[0:10]}"
 
 
 def skip_by_tag(r, include_tags, exclude_tags):
@@ -881,16 +852,13 @@ def confirm_payment(
     pay_denom = "".join([i for i in tx_amount if not i.isdigit()])
     if pay_denom != "alnt":
         logger.log(
-            f"{record.id}: {pay_denom} in tx {tx.hash} is not an expected "
-            "payment denomination"
+            f"{record.id}: {pay_denom} in tx {tx.hash} is not an expected " "payment denomination"
         )
         return False
 
     pay_amount = int("".join([i for i in tx_amount if i.isdigit()]) or "0")
     if pay_amount < min_amount:
-        logger.log(
-            f"{record.id}: payment amount {tx.amount} is less than minimum {min_amount}"
-        )
+        logger.log(f"{record.id}: payment amount {tx.amount} is less than minimum {min_amount}")
         return False
 
     # Check if the payment was already used on a deployment
@@ -914,9 +882,7 @@ def confirm_payment(
         {"deployer": record.attributes.deployer, "payment": tx.hash}, all=True
     )
     if len(used):
-        logger.log(
-            f"{record.id}: payment {tx.hash} already used on deployment removal {used}"
-        )
+        logger.log(f"{record.id}: payment {tx.hash} already used on deployment removal {used}")
         return False
 
     return True
@@ -940,9 +906,7 @@ def confirm_auction(
 
     # Cross check app against application in the auction record
     requested_app = laconic.get_record(record.attributes.application, require=True)
-    auction_app = laconic.get_record(
-        auction_records_by_id[0].attributes.application, require=True
-    )
+    auction_app = laconic.get_record(auction_records_by_id[0].attributes.application, require=True)
     requested_app_id = requested_app.id if requested_app else None
     auction_app_id = auction_app.id if auction_app else None
     if requested_app_id != auction_app_id:

stack_orchestrator/main.py

@@ -15,30 +15,24 @@
 
 import click
 
+from stack_orchestrator import opts, update, version
+from stack_orchestrator.build import build_containers, build_npms, build_webapp, fetch_containers
 from stack_orchestrator.command_types import CommandOptions
-from stack_orchestrator.repos import setup_repositories
-from stack_orchestrator.repos import fetch_stack
-from stack_orchestrator.build import build_containers, fetch_containers
-from stack_orchestrator.build import build_npms
-from stack_orchestrator.build import build_webapp
+from stack_orchestrator.deploy import deploy, deployment
 from stack_orchestrator.deploy.webapp import (
-    run_webapp,
     deploy_webapp,
     deploy_webapp_from_registry,
-    undeploy_webapp_from_registry,
-    publish_webapp_deployer,
-    publish_deployment_auction,
     handle_deployment_auction,
+    publish_deployment_auction,
+    publish_webapp_deployer,
     request_webapp_deployment,
     request_webapp_undeployment,
+    run_webapp,
+    undeploy_webapp_from_registry,
 )
-from stack_orchestrator.deploy import deploy
-from stack_orchestrator import version
-from stack_orchestrator.deploy import deployment
-from stack_orchestrator import opts
-from stack_orchestrator import update
+from stack_orchestrator.repos import fetch_stack, setup_repositories
 
-CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
+CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]}
 
 
 @click.group(context_settings=CONTEXT_SETTINGS)

stack_orchestrator/repos/fetch_stack.py

@@ -17,9 +17,9 @@
 # CERC_REPO_BASE_DIR defaults to ~/cerc
 
 
-import click
 import os
 
+import click
 from decouple import config
 from git import exc
 
@@ -36,9 +36,7 @@ from stack_orchestrator.util import error_exit
 @click.pass_context
 def command(ctx, stack_locator, git_ssh, check_only, pull):
     """Optionally resolve then git clone a repository with stack definitions."""
-    dev_root_path = os.path.expanduser(
-        str(config("CERC_REPO_BASE_DIR", default="~/cerc"))
-    )
+    dev_root_path = os.path.expanduser(str(config("CERC_REPO_BASE_DIR", default="~/cerc")))
     if not opts.o.quiet:
         print(f"Dev Root is: {dev_root_path}")
     try:

stack_orchestrator/repos/setup_repositories.py

@@ -16,20 +16,22 @@
 # env vars:
 # CERC_REPO_BASE_DIR defaults to ~/cerc
 
+import importlib.resources
 import os
 import sys
-from decouple import config
-import git
-from git.exc import GitCommandError, InvalidGitRepositoryError
 from typing import Any
-from tqdm import tqdm
+
 import click
-import importlib.resources
+import git
+from decouple import config
+from git.exc import GitCommandError, InvalidGitRepositoryError
+from tqdm import tqdm
+
 from stack_orchestrator.opts import opts
 from stack_orchestrator.util import (
+    error_exit,
     get_parsed_stack_config,
     include_exclude_check,
-    error_exit,
     warn_exit,
 )
 
@@ -86,48 +88,38 @@ def _get_repo_current_branch_or_tag(full_filesystem_repo_path):
     current_repo_branch_or_tag = "***UNDETERMINED***"
     is_branch = False
     try:
-        current_repo_branch_or_tag = git.Repo(
-            full_filesystem_repo_path
-        ).active_branch.name
+        current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).active_branch.name
         is_branch = True
     except TypeError:
         # This means that the current ref is not a branch, so possibly a tag
         # Let's try to get the tag
         try:
-            current_repo_branch_or_tag = git.Repo(
-                full_filesystem_repo_path
-            ).git.describe("--tags", "--exact-match")
+            current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).git.describe(
+                "--tags", "--exact-match"
+            )
             # Note that git is asymmetric -- the tag you told it to check out
             # may not be the one you get back here (if there are multiple tags
             # associated with the same commit)
         except GitCommandError:
             # If there is no matching branch or tag checked out, just use the current
             # SHA
-            current_repo_branch_or_tag = (
-                git.Repo(full_filesystem_repo_path).commit("HEAD").hexsha
-            )
+            current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).commit("HEAD").hexsha
     return current_repo_branch_or_tag, is_branch
 
 
 # TODO: fix the messy arg list here
-def process_repo(
-    pull, check_only, git_ssh, dev_root_path, branches_array, fully_qualified_repo
-):
+def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully_qualified_repo):
     if opts.o.verbose:
         print(f"Processing repo: {fully_qualified_repo}")
     repo_host, repo_path, repo_branch = host_and_path_for_repo(fully_qualified_repo)
     git_ssh_prefix = f"git@{repo_host}:"
     git_http_prefix = f"https://{repo_host}/"
-    full_github_repo_path = (
-        f"{git_ssh_prefix if git_ssh else git_http_prefix}{repo_path}"
-    )
+    full_github_repo_path = f"{git_ssh_prefix if git_ssh else git_http_prefix}{repo_path}"
     repoName = repo_path.split("/")[-1]
     full_filesystem_repo_path = os.path.join(dev_root_path, repoName)
     is_present = os.path.isdir(full_filesystem_repo_path)
     (current_repo_branch_or_tag, is_branch) = (
-        _get_repo_current_branch_or_tag(full_filesystem_repo_path)
-        if is_present
-        else (None, None)
+        _get_repo_current_branch_or_tag(full_filesystem_repo_path) if is_present else (None, None)
     )
     if not opts.o.quiet:
         present_text = (
@@ -140,10 +132,7 @@ def process_repo(
     # Quick check that it's actually a repo
     if is_present:
         if not is_git_repo(full_filesystem_repo_path):
-            print(
-                f"Error: {full_filesystem_repo_path} does not contain "
-                "a valid git repository"
-            )
+            print(f"Error: {full_filesystem_repo_path} does not contain " "a valid git repository")
             sys.exit(1)
         else:
             if pull:
@@ -190,8 +179,7 @@ def process_repo(
 
     if branch_to_checkout:
         if current_repo_branch_or_tag is None or (
-            current_repo_branch_or_tag
-            and (current_repo_branch_or_tag != branch_to_checkout)
+            current_repo_branch_or_tag and (current_repo_branch_or_tag != branch_to_checkout)
         ):
             if not opts.o.quiet:
                 print(f"switching to branch {branch_to_checkout} in repo {repo_path}")
@@ -245,14 +233,9 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches):
 
     if local_stack:
         dev_root_path = os.getcwd()[0 : os.getcwd().rindex("stack-orchestrator")]
-        print(
-            f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: "
-            f"{dev_root_path}"
-        )
+        print(f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: " f"{dev_root_path}")
     else:
-        dev_root_path = os.path.expanduser(
-            str(config("CERC_REPO_BASE_DIR", default="~/cerc"))
-        )
+        dev_root_path = os.path.expanduser(str(config("CERC_REPO_BASE_DIR", default="~/cerc")))
 
     if not quiet:
         print(f"Dev Root is: {dev_root_path}")
@@ -265,9 +248,7 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches):
     # See: https://stackoverflow.com/a/20885799/1701505
     from stack_orchestrator import data
 
-    with importlib.resources.open_text(
-        data, "repository-list.txt"
-    ) as repository_list_file:
+    with importlib.resources.open_text(data, "repository-list.txt") as repository_list_file:
         all_repos = repository_list_file.read().splitlines()
 
     repos_in_scope = []

stack_orchestrator/update.py

@@ -13,16 +13,18 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-import click
 import datetime
 import filecmp
 import os
-from pathlib import Path
-import requests
-import sys
-import stat
 import shutil
+import stat
+import sys
+from pathlib import Path
+
+import click
+import requests
 import validators
+
 from stack_orchestrator.util import get_yaml
 
 
@@ -40,9 +42,7 @@ def _error_exit(s: str):
 
 # Note at present this probably won't work on non-Unix based OSes like Windows
 @click.command()
-@click.option(
-    "--check-only", is_flag=True, default=False, help="only check, don't update"
-)
+@click.option("--check-only", is_flag=True, default=False, help="only check, don't update")
 @click.pass_context
 def command(ctx, check_only):
     """update shiv binary from a distribution url"""
@@ -52,7 +52,7 @@ def command(ctx, check_only):
     if not config_file_path.exists():
         _error_exit(f"Error: Config file: {config_file_path} not found")
     yaml = get_yaml()
-    config = yaml.load(open(config_file_path, "r"))
+    config = yaml.load(open(config_file_path))
     if "distribution-url" not in config:
         _error_exit(f"Error: {config_key} not defined in {config_file_path}")
     distribution_url = config[config_key]
@@ -61,9 +61,7 @@ def command(ctx, check_only):
         _error_exit(f"ERROR: distribution url: {distribution_url} is not valid")
     # Figure out the filename for ourselves
     shiv_binary_path = Path(sys.argv[0])
-    timestamp_filename = (
-        f"laconic-so-download-{datetime.datetime.now().strftime('%y%m%d-%H%M%S')}"
-    )
+    timestamp_filename = f"laconic-so-download-{datetime.datetime.now().strftime('%y%m%d-%H%M%S')}"
     temp_download_path = shiv_binary_path.parent.joinpath(timestamp_filename)
     # Download the file to a temp filename
     if ctx.obj.verbose:

stack_orchestrator/util.py

@@ -13,14 +13,17 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from decouple import config
 import os.path
 import sys
-import ruamel.yaml
+from collections.abc import Mapping
 from pathlib import Path
+from typing import NoReturn
+
+import ruamel.yaml
+from decouple import config
 from dotenv import dotenv_values
-from typing import Mapping, NoReturn, Optional, Set, List
-from stack_orchestrator.constants import stack_file_name, deployment_file_name
+
+from stack_orchestrator.constants import deployment_file_name, stack_file_name
 
 
 def include_exclude_check(s, include, exclude):
@@ -50,14 +53,9 @@ def get_dev_root_path(ctx):
     if ctx and ctx.local_stack:
         # TODO: This code probably doesn't work
         dev_root_path = os.getcwd()[0 : os.getcwd().rindex("stack-orchestrator")]
-        print(
-            f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: "
-            f"{dev_root_path}"
-        )
+        print(f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: " f"{dev_root_path}")
     else:
-        dev_root_path = os.path.expanduser(
-            str(config("CERC_REPO_BASE_DIR", default="~/cerc"))
-        )
+        dev_root_path = os.path.expanduser(str(config("CERC_REPO_BASE_DIR", default="~/cerc")))
     return dev_root_path
 
 
@@ -65,7 +63,7 @@ def get_dev_root_path(ctx):
 def get_parsed_stack_config(stack):
     stack_file_path = get_stack_path(stack).joinpath(stack_file_name)
     if stack_file_path.exists():
-        return get_yaml().load(open(stack_file_path, "r"))
+        return get_yaml().load(open(stack_file_path))
     # We try here to generate a useful diagnostic error
     # First check if the stack directory is present
     if stack_file_path.parent.exists():
@@ -75,8 +73,6 @@ def get_parsed_stack_config(stack):
 
 def get_pod_list(parsed_stack):
     # Handle both old and new format
-    if "pods" not in parsed_stack or not parsed_stack["pods"]:
-        return []
     pods = parsed_stack["pods"]
     if type(pods[0]) is str:
         result = pods
@@ -103,10 +99,10 @@ def get_job_list(parsed_stack):
     return result
 
 
-def get_plugin_code_paths(stack) -> List[Path]:
+def get_plugin_code_paths(stack) -> list[Path]:
     parsed_stack = get_parsed_stack_config(stack)
-    pods = parsed_stack.get("pods") or []
-    result: Set[Path] = set()
+    pods = parsed_stack["pods"]
+    result: set[Path] = set()
     for pod in pods:
         if type(pod) is str:
             result.add(get_stack_path(stack))
@@ -155,16 +151,15 @@ def resolve_job_compose_file(stack, job_name: str):
         if proposed_file.exists():
             return proposed_file
         # If we don't find it fall through to the internal case
-    data_dir = Path(__file__).absolute().parent.joinpath("data")
-    compose_jobs_base = data_dir.joinpath("compose-jobs")
+    # TODO: Add internal compose-jobs directory support if needed
+    # For now, jobs are expected to be in external stacks only
+    compose_jobs_base = Path(stack).parent.parent.joinpath("compose-jobs")
     return compose_jobs_base.joinpath(f"docker-compose-{job_name}.yml")
 
 
 def get_pod_file_path(stack, parsed_stack, pod_name: str):
-    pods = parsed_stack.get("pods") or []
+    pods = parsed_stack["pods"]
     result = None
-    if not pods:
-        return result
     if type(pods[0]) is str:
         result = resolve_compose_file(stack, pod_name)
     else:
@@ -192,9 +187,9 @@ def get_job_file_path(stack, parsed_stack, job_name: str):
 
 
 def get_pod_script_paths(parsed_stack, pod_name: str):
-    pods = parsed_stack.get("pods") or []
+    pods = parsed_stack["pods"]
     result = []
-    if not pods or not type(pods[0]) is str:
+    if type(pods[0]) is not str:
         for pod in pods:
             if pod["name"] == pod_name:
                 pod_root_dir = os.path.join(
@@ -210,9 +205,9 @@ def get_pod_script_paths(parsed_stack, pod_name: str):
 
 
 def pod_has_scripts(parsed_stack, pod_name: str):
-    pods = parsed_stack.get("pods") or []
+    pods = parsed_stack["pods"]
     result = False
-    if not pods or type(pods[0]) is str:
+    if type(pods[0]) is str:
         result = False
     else:
         for pod in pods:
@@ -246,7 +241,7 @@ def get_k8s_dir():
 def get_parsed_deployment_spec(spec_file):
     spec_file_path = Path(spec_file)
     try:
-        return get_yaml().load(open(spec_file_path, "r"))
+        return get_yaml().load(open(spec_file_path))
     except FileNotFoundError as error:
         # We try here to generate a useful diagnostic error
         print(f"Error: spec file: {spec_file_path} does not exist")
@@ -296,5 +291,6 @@ def warn_exit(s) -> NoReturn:
     sys.exit(0)
 
 
-def env_var_map_from_file(file: Path) -> Mapping[str, Optional[str]]:
-    return dotenv_values(file)
+def env_var_map_from_file(file: Path) -> Mapping[str, str | None]:
+    result: Mapping[str, str | None] = dotenv_values(file)
+    return result

stack_orchestrator/version.py

@@ -13,8 +13,9 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
+from importlib import metadata, resources
+
 import click
-from importlib import resources, metadata
 
 
 @click.command()

tests/k8s-deploy/run-deploy-test.sh

@@ -105,15 +105,6 @@ fi
 # Add a config file to be picked up by the ConfigMap before starting.
 echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/configmaps/test-config/test_config
 
-# Add secrets to the deployment spec (references a pre-existing k8s Secret by name).
-# deploy init already writes an empty 'secrets: {}' key, so we replace it
-# rather than appending (ruamel.yaml rejects duplicate keys).
-deployment_spec_file=${test_deployment_dir}/spec.yml
-sed -i 's/^secrets: {}$/secrets:\n  test-secret:\n    - TEST_SECRET_KEY/' ${deployment_spec_file}
-
-# Get the deployment ID for kubectl queries
-deployment_id=$(cat ${test_deployment_dir}/deployment.yml | cut -d ' ' -f 2)
-
 echo "deploy create output file test: passed"
 # Try to start the deployment
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
@@ -175,71 +166,12 @@ else
     delete_cluster_exit
 fi
 
-# --- New feature tests: namespace, labels, jobs, secrets ---
-
-# Check that the pod is in the deployment-specific namespace (not default)
-ns_pod_count=$(kubectl get pods -n laconic-${deployment_id} -l app=${deployment_id} --no-headers 2>/dev/null | wc -l)
-if [ "$ns_pod_count" -gt 0 ]; then
-    echo "namespace isolation test: passed"
-else
-    echo "namespace isolation test: FAILED"
-    echo "Expected pod in namespace laconic-${deployment_id}"
-    delete_cluster_exit
-fi
-
-# Check that the stack label is set on the pod
-stack_label_count=$(kubectl get pods -n laconic-${deployment_id} -l app.kubernetes.io/stack=test --no-headers 2>/dev/null | wc -l)
-if [ "$stack_label_count" -gt 0 ]; then
-    echo "stack label test: passed"
-else
-    echo "stack label test: FAILED"
-    delete_cluster_exit
-fi
-
-# Check that the job completed successfully
-for i in {1..30}; do
-    job_status=$(kubectl get job ${deployment_id}-job-test-job -n laconic-${deployment_id} -o jsonpath='{.status.succeeded}' 2>/dev/null || true)
-    if [ "$job_status" == "1" ]; then
-        break
-    fi
-    sleep 2
-done
-if [ "$job_status" == "1" ]; then
-    echo "job completion test: passed"
-else
-    echo "job completion test: FAILED"
-    echo "Job status.succeeded: ${job_status}"
-    delete_cluster_exit
-fi
-
-# Check that the secrets spec results in an envFrom secretRef on the pod
-secret_ref=$(kubectl get pod -n laconic-${deployment_id} -l app=${deployment_id} \
-    -o jsonpath='{.items[0].spec.containers[0].envFrom[?(@.secretRef.name=="test-secret")].secretRef.name}' 2>/dev/null || true)
-if [ "$secret_ref" == "test-secret" ]; then
-    echo "secrets envFrom test: passed"
-else
-    echo "secrets envFrom test: FAILED"
-    echo "Expected secretRef 'test-secret', got: ${secret_ref}"
-    delete_cluster_exit
-fi
-
-# Stop then start again and check the volume was preserved.
-# Use --skip-cluster-management to reuse the existing kind cluster instead of
-# destroying and recreating it (which fails on CI runners due to stale etcd/certs
-# and cgroup detection issues).
-# Use --delete-volumes to clear PVs so fresh PVCs can bind on restart.
-# Bind-mount data survives on the host filesystem; provisioner volumes are recreated fresh.
-$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes --skip-cluster-management
-# Wait for the namespace to be fully terminated before restarting.
-# Without this, 'start' fails with 403 Forbidden because the namespace
-# is still in Terminating state.
-for i in {1..60}; do
-    if ! kubectl get namespace laconic-${deployment_id} 2>/dev/null | grep -q .; then
-        break
-    fi
-    sleep 2
-done
-$TEST_TARGET_SO deployment --dir $test_deployment_dir start --skip-cluster-management
+# Stop then start again and check the volume was preserved
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
+# Sleep a bit just in case
+# sleep for longer to check if that's why the subsequent create cluster fails
+sleep 20
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
 wait_for_pods_started
 wait_for_log_output
 sleep 1
@@ -252,9 +184,8 @@ else
     delete_cluster_exit
 fi
 
-# Provisioner volumes are destroyed when PVs are deleted (--delete-volumes on stop).
-# Unlike bind-mount volumes whose data persists on the host, provisioner storage
-# is gone, so the volume appears fresh after restart.
+# These volumes will be completely destroyed by the kind delete/create, because they lived inside
+# the kind container.  So, unlike the bind-mount case, they will appear fresh after the restart.
 log_output_11=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
 if [[ "$log_output_11" == *"/data2 filesystem is fresh"* ]]; then
     echo "Fresh provisioner volumes test: passed"

tests/k8s-deployment-control/run-test.sh

@@ -206,7 +206,7 @@ fi
 # The deployment's pod should be scheduled onto node: worker3
 # Check that's what happened
 # Get get the node onto which the stack pod has been deployed
-deployment_node=$(kubectl get pods -n laconic-${deployment_id} -l app=${deployment_id} -o=jsonpath='{.items..spec.nodeName}')
+deployment_node=$(kubectl get pods -l app=${deployment_id} -o=jsonpath='{.items..spec.nodeName}')
 expected_node=${deployment_id}-worker3
 echo "Stack pod deployed to node: ${deployment_node}"
 if [[ ${deployment_node} == ${expected_node} ]]; then

tests/scripts/install-kind.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 # TODO: handle ARM
-curl --silent -Lo ./kind https://kind.sigs.k8s.io/dl/v0.25.0/kind-linux-amd64
+curl --silent -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
 chmod +x ./kind
 mv ./kind /usr/local/bin

tests/scripts/install-kubectl.sh

@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
 # TODO: handle ARM
-# Pin kubectl to match Kind's default k8s version (v1.31.x)
-curl --silent -LO "https://dl.k8s.io/release/v1.31.2/bin/linux/amd64/kubectl"
+curl --silent -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
 chmod +x ./kubectl
 mv ./kubectl /usr/local/bin