fix(k8s): drop stale ACME accounts during etcd cleanup

_clean_etcd_keeping_certs() preserved ALL caddy-system secrets across
cluster recreations, including ACME account secrets registered with
wrong/empty email. Caddy reuses these stale accounts instead of
registering fresh ones, causing recurring "unable to parse email
address" errors.

Filter the etcd restore loop to only keep certificate secrets (keys
matching *certificates*). ACME accounts, OCSP staples, and locks are
transient and get recreated automatically by Caddy on startup.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

stack_orchestrator/constants.py

@@ -45,4 +45,3 @@ runtime_class_key = "runtime-class"
 high_memlock_runtime = "high-memlock"
 high_memlock_spec_filename = "high-memlock-spec.json"
 acme_email_key = "acme-email"
-kind_mount_root_key = "kind-mount-root"

stack_orchestrator/deploy/k8s/cluster_info.py

@@ -371,11 +371,7 @@ class ClusterInfo:
 
             if self.spec.is_kind_deployment():
                 host_path = client.V1HostPathVolumeSource(
-                    path=get_kind_pv_bind_mount_path(
-                        volume_name,
-                        kind_mount_root=self.spec.get_kind_mount_root(),
-                        host_path=volume_path,
-                    )
+                    path=get_kind_pv_bind_mount_path(volume_name)
                 )
             else:
                 host_path = client.V1HostPathVolumeSource(path=volume_path)
@@ -398,14 +394,6 @@ class ClusterInfo:
             result.append(pv)
         return result
 
-    def _any_service_has_host_network(self):
-        for pod_name in self.parsed_pod_yaml_map:
-            pod = self.parsed_pod_yaml_map[pod_name]
-            for svc in pod.get("services", {}).values():
-                if svc.get("network_mode") == "host":
-                    return True
-        return False
-
     # TODO: put things like image pull policy into an object-scope struct
     def get_deployment(self, image_pull_policy: Optional[str] = None):
         containers = []
@@ -580,7 +568,6 @@ class ClusterInfo:
                     )
                 )
 
-        use_host_network = self._any_service_has_host_network()
         template = client.V1PodTemplateSpec(
             metadata=client.V1ObjectMeta(annotations=annotations, labels=labels),
             spec=client.V1PodSpec(
@@ -590,10 +577,6 @@ class ClusterInfo:
                 affinity=affinity,
                 tolerations=tolerations,
                 runtime_class_name=self.spec.get_runtime_class(),
-                host_network=use_host_network or None,
-                dns_policy=(
-                    "ClusterFirstWithHostNet" if use_host_network else None
-                ),
             ),
         )
         spec = client.V1DeploymentSpec(

stack_orchestrator/deploy/k8s/deploy_k8s.py

@@ -574,14 +574,14 @@ class K8sDeployer(Deployer):
 
     def logs(self, services, tail, follow, stream):
         self.connect_api()
-        pods = pods_in_deployment(self.core_api, self.cluster_info.app_name, namespace=self.k8s_namespace)
+        pods = pods_in_deployment(self.core_api, self.cluster_info.app_name)
         if len(pods) > 1:
             print("Warning: more than one pod in the deployment")
         if len(pods) == 0:
             log_data = "******* Pods not running ********\n"
         else:
             k8s_pod_name = pods[0]
-            containers = containers_in_pod(self.core_api, k8s_pod_name, namespace=self.k8s_namespace)
+            containers = containers_in_pod(self.core_api, k8s_pod_name)
             # If pod not started, logs request below will throw an exception
             try:
                 log_data = ""

stack_orchestrator/deploy/k8s/helpers.py

@@ -118,12 +118,17 @@ def _get_etcd_host_path_from_kind_config(config_file: str) -> Optional[str]:
 
 
 def _clean_etcd_keeping_certs(etcd_path: str) -> bool:
-    """Clean persisted etcd, keeping only TLS certificates.
+    """Clean persisted etcd, keeping only TLS certificate secrets.
 
     When etcd is persisted and a cluster is recreated, kind tries to install
     resources fresh but they already exist. Instead of trying to delete
     specific stale resources (blacklist), we keep only the valuable data
-    (caddy TLS certs) and delete everything else (whitelist approach).
+    (caddy TLS certificate secrets) and delete everything else.
+
+    ACME account secrets are intentionally NOT preserved — they may contain
+    stale registrations (e.g. with wrong/empty email) that cause "unable to
+    parse email address" errors. Caddy re-registers accounts automatically
+    on startup using the email from the ConfigMap.
 
     The etcd image is distroless (no shell), so we extract the statically-linked
     etcdctl binary and run it from alpine which has shell support.
@@ -218,6 +223,11 @@ def _clean_etcd_keeping_certs(etcd_path: str) -> bool:
                 jq -r ".kvs[] | @base64" /backup/kept.json 2>/dev/null | \
                 while read encoded; do
                     key=$(echo $encoded | base64 -d | jq -r ".key" | base64 -d)
+                    # Keep only certificate secrets, drop ACME accounts/locks/OCSP
+                    case "$key" in
+                        *certificates*) ;;
+                        *) continue ;;
+                    esac
                     val=$(echo $encoded | base64 -d | jq -r ".value" | base64 -d)
                     echo "$val" | /backup/etcdctl put "$key"
                 done
@@ -259,7 +269,7 @@ def _clean_etcd_keeping_certs(etcd_path: str) -> bool:
         return False
 
     if opts.o.debug:
-        print("Cleaned etcd, kept only TLS certificates")
+        print("Cleaned etcd, kept only TLS certificate secrets (dropped ACME accounts)")
     return True
 
 
@@ -393,10 +403,10 @@ def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]):
             raise DeployerException(f"kind load docker-image failed: {result}")
 
 
-def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str, namespace: str = "default"):
+def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
     pods = []
     pod_response = core_api.list_namespaced_pod(
-        namespace=namespace, label_selector=f"app={deployment_name}"
+        namespace="default", label_selector=f"app={deployment_name}"
     )
     if opts.o.debug:
         print(f"pod_response: {pod_response}")
@@ -406,10 +416,10 @@ def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str, namespa
     return pods
 
 
-def containers_in_pod(core_api: client.CoreV1Api, pod_name: str, namespace: str = "default") -> List[str]:
+def containers_in_pod(core_api: client.CoreV1Api, pod_name: str) -> List[str]:
     containers: List[str] = []
     pod_response = cast(
-        client.V1Pod, core_api.read_namespaced_pod(pod_name, namespace=namespace)
+        client.V1Pod, core_api.read_namespaced_pod(pod_name, namespace="default")
     )
     if opts.o.debug:
         print(f"pod_response: {pod_response}")
@@ -440,11 +450,7 @@ def named_volumes_from_pod_files(parsed_pod_files):
     return named_volumes
 
 
-def get_kind_pv_bind_mount_path(volume_name: str, kind_mount_root: Optional[str] = None,
-                                host_path: Optional[str] = None):
-    if kind_mount_root and host_path and host_path.startswith(kind_mount_root):
-        rel = os.path.relpath(host_path, kind_mount_root)
-        return f"/mnt/{rel}"
+def get_kind_pv_bind_mount_path(volume_name: str):
     return f"/mnt/{volume_name}"
 
 
@@ -567,7 +573,6 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context):
     volume_definitions = []
     volume_host_path_map = _get_host_paths_for_volumes(deployment_context)
     seen_host_path_mounts = set()  # Track to avoid duplicate mounts
-    kind_mount_root = deployment_context.spec.get_kind_mount_root()
 
     # Cluster state backup for offline data recovery (unique per deployment)
     # etcd contains all k8s state; PKI certs needed to decrypt etcd offline
@@ -588,17 +593,6 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context):
         f"  - hostPath: {pki_host_path}\n" f"    containerPath: /etc/kubernetes/pki\n"
     )
 
-    # When kind-mount-root is set, emit a single extraMount for the root.
-    # Individual volumes whose host path starts with the root are covered
-    # by this single mount and don't need their own extraMount entries.
-    mount_root_emitted = False
-    if kind_mount_root:
-        volume_definitions.append(
-            f"  - hostPath: {kind_mount_root}\n"
-            f"    containerPath: /mnt\n"
-        )
-        mount_root_emitted = True
-
     # Note these paths are relative to the location of the pod files (at present)
     # So we need to fix up to make them correct and absolute because kind assumes
     # relative to the cwd.
@@ -658,9 +652,6 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context):
                                         volume_host_path_map[volume_name],
                                         deployment_dir,
                                     )
-                                    # Skip individual extraMount if covered by mount root
-                                    if mount_root_emitted and str(host_path).startswith(kind_mount_root):
-                                        continue
                                     container_path = get_kind_pv_bind_mount_path(
                                         volume_name
                                     )
@@ -702,35 +693,11 @@ def _generate_kind_port_mappings_from_services(parsed_pod_files):
 
 def _generate_kind_port_mappings(parsed_pod_files):
     port_definitions = []
-    seen = set()
     # Map port 80 and 443 for the Caddy ingress controller (HTTPS support)
     for port_string in ["80", "443"]:
         port_definitions.append(
             f"  - containerPort: {port_string}\n    hostPort: {port_string}\n"
         )
-        seen.add((port_string, "TCP"))
-    # Map ports declared in compose services
-    for pod in parsed_pod_files:
-        parsed_pod_file = parsed_pod_files[pod]
-        if "services" in parsed_pod_file:
-            for service_name in parsed_pod_file["services"]:
-                service_obj = parsed_pod_file["services"][service_name]
-                for port_entry in service_obj.get("ports", []):
-                    port_str = str(port_entry)
-                    protocol = "TCP"
-                    if "/" in port_str:
-                        port_str, proto = port_str.split("/", 1)
-                        protocol = proto.upper()
-                    if ":" in port_str:
-                        port_str = port_str.split(":")[-1]
-                    port_num = port_str.strip("'\"")
-                    if (port_num, protocol) not in seen:
-                        seen.add((port_num, protocol))
-                        port_definitions.append(
-                            f"  - containerPort: {port_num}\n"
-                            f"    hostPort: {port_num}\n"
-                            f"    protocol: {protocol}\n"
-                        )
     return (
         ""
         if len(port_definitions) == 0

stack_orchestrator/deploy/spec.py

@@ -202,8 +202,5 @@ class Spec:
     def is_kind_deployment(self):
         return self.get_deployment_type() in [constants.k8s_kind_deploy_type]
 
-    def get_kind_mount_root(self):
-        return self.obj.get(constants.kind_mount_root_key)
-
     def is_docker_deployment(self):
         return self.get_deployment_type() in [constants.compose_deploy_type]

tests/scripts/run-test-local.sh

@@ -1,53 +0,0 @@
-#!/bin/bash
-# Run a test suite locally in an isolated venv.
-#
-# Usage:
-#   ./tests/scripts/run-test-local.sh <test-script>
-#
-# Examples:
-#   ./tests/scripts/run-test-local.sh tests/webapp-test/run-webapp-test.sh
-#   ./tests/scripts/run-test-local.sh tests/smoke-test/run-smoke-test.sh
-#   ./tests/scripts/run-test-local.sh tests/k8s-deploy/run-deploy-test.sh
-#
-# The script creates a temporary venv, installs shiv, builds the laconic-so
-# package, runs the requested test, then cleans up.
-
-set -euo pipefail
-
-if [ $# -lt 1 ]; then
-  echo "Usage: $0 <test-script> [args...]"
-  exit 1
-fi
-
-TEST_SCRIPT="$1"
-shift
-
-if [ ! -f "$TEST_SCRIPT" ]; then
-  echo "Error: $TEST_SCRIPT not found"
-  exit 1
-fi
-
-REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-VENV_DIR=$(mktemp -d /tmp/so-test-XXXXXX)
-
-cleanup() {
-  echo "Cleaning up venv: $VENV_DIR"
-  rm -rf "$VENV_DIR"
-}
-trap cleanup EXIT
-
-cd "$REPO_DIR"
-
-echo "==> Creating venv in $VENV_DIR"
-python3 -m venv "$VENV_DIR"
-source "$VENV_DIR/bin/activate"
-
-echo "==> Installing shiv"
-pip install -q shiv
-
-echo "==> Building laconic-so package"
-./scripts/create_build_tag_file.sh
-./scripts/build_shiv_package.sh
-
-echo "==> Running: $TEST_SCRIPT $*"
-exec "./$TEST_SCRIPT" "$@"