From 7acabb0743611e7f2015e24dc4398dfccd3e3739 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Thu, 27 Nov 2025 06:43:07 +0000 Subject: [PATCH] Add support for generating Helm charts when creating a deployment (#974) Part of https://plan.wireit.in/deepstack/browse/VUL-265/ - Added a flag `--helm-chart` to `deploy create` command - Uses Kompose CLI wrapper to generate a helm chart from compose files in a stack - To be handled in a follow on PR(s): - Templatize generated charts and generate a `values.yml` file with defaults Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/974 Co-authored-by: Prathamesh Musale Co-committed-by: Prathamesh Musale --- docs/helm-chart-generation.md | 113 ++++++++ .../deploy/deployment_create.py | 15 +- .../deploy/k8s/helm/__init__.py | 14 + .../deploy/k8s/helm/chart_generator.py | 266 ++++++++++++++++++ .../deploy/k8s/helm/kompose_wrapper.py | 109 +++++++ .../deploy/webapp/deploy_webapp.py | 1 + 6 files changed, 515 insertions(+), 3 deletions(-) create mode 100644 docs/helm-chart-generation.md create mode 100644 stack_orchestrator/deploy/k8s/helm/__init__.py create mode 100644 stack_orchestrator/deploy/k8s/helm/chart_generator.py create mode 100644 stack_orchestrator/deploy/k8s/helm/kompose_wrapper.py diff --git a/docs/helm-chart-generation.md b/docs/helm-chart-generation.md new file mode 100644 index 00000000..903ae2da --- /dev/null +++ b/docs/helm-chart-generation.md @@ -0,0 +1,113 @@ +# Helm Chart Generation + +Generate Kubernetes Helm charts from stack compose files using Kompose. + +## Prerequisites + +Install Kompose: + +```bash +# Linux +curl -L https://github.com/kubernetes/kompose/releases/download/v1.34.0/kompose-linux-amd64 -o kompose +chmod +x kompose +sudo mv kompose /usr/local/bin/ + +# macOS +brew install kompose + +# Verify +kompose version +``` + +## Usage + +### 1. Create spec file + +```bash +laconic-so --stack deploy --deploy-to k8s init \ + --kube-config ~/.kube/config \ + --output spec.yml +``` + +### 2. Generate Helm chart + +```bash +laconic-so --stack deploy create \ + --spec-file spec.yml \ + --deployment-dir my-deployment \ + --helm-chart +``` + +### 3. Deploy to Kubernetes + +```bash +helm install my-release my-deployment/chart +kubectl get pods -n zenith +``` + +## Output Structure + +```bash +my-deployment/ +├── spec.yml # Reference +├── stack.yml # Reference +└── chart/ # Helm chart + ├── Chart.yaml + ├── README.md + └── templates/ + └── *.yaml +``` + +## Example + +```bash +# Generate chart for stage1-zenithd +laconic-so --stack stage1-zenithd deploy --deploy-to k8s init \ + --kube-config ~/.kube/config \ + --output stage1-spec.yml + +laconic-so --stack stage1-zenithd deploy create \ + --spec-file stage1-spec.yml \ + --deployment-dir stage1-deployment \ + --helm-chart + +# Deploy +helm install stage1-zenithd stage1-deployment/chart +``` + +## Production Deployment (TODO) + +### Local Development + +```bash +# Access services using port-forward +kubectl port-forward service/zenithd 26657:26657 +kubectl port-forward service/nginx-api-proxy 1317:80 +kubectl port-forward service/cosmos-explorer 4173:4173 +``` + +### Production Access Options + +- Option 1: Ingress + cert-manager (Recommended) + - Install ingress-nginx + cert-manager + - Point DNS to cluster LoadBalancer IP + - Auto-provisions Let's Encrypt TLS certs + - Access: `https://api.zenith.example.com` +- Option 2: Cloud LoadBalancer + - Use cloud provider's LoadBalancer service type + - Point DNS to assigned external IP + - Manual TLS cert management +- Option 3: Bare Metal (MetalLB + Ingress) + - MetalLB provides LoadBalancer IPs from local network + - Same Ingress setup as cloud +- Option 4: NodePort + External Proxy + - Expose services on 30000-32767 range + - External nginx/Caddy proxies 80/443 → NodePort + - Manual cert management + +### Changes Needed + +- Add Ingress template to charts +- Add TLS configuration to values.yaml +- Document cert-manager setup +- Add production deployment guide diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index 9d45f226..0b3a92f7 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -443,22 +443,31 @@ def _check_volume_definitions(spec): @click.command() @click.option("--spec-file", required=True, help="Spec file to use to create this deployment") @click.option("--deployment-dir", help="Create deployment files in this directory") +@click.option("--helm-chart", is_flag=True, default=False, help="Generate Helm chart instead of deploying (k8s only)") # TODO: Hack @click.option("--network-dir", help="Network configuration supplied in this directory") @click.option("--initial-peers", help="Initial set of persistent peers") @click.pass_context -def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): +def create(ctx, spec_file, deployment_dir, helm_chart, network_dir, initial_peers): deployment_command_context = ctx.obj - return create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers) + return create_operation(deployment_command_context, spec_file, deployment_dir, helm_chart, network_dir, initial_peers) # The init command's implementation is in a separate function so that we can # call it from other commands, bypassing the click decoration stuff -def create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers): +def create_operation(deployment_command_context, spec_file, deployment_dir, helm_chart, network_dir, initial_peers): parsed_spec = Spec(os.path.abspath(spec_file), get_parsed_deployment_spec(spec_file)) _check_volume_definitions(parsed_spec) stack_name = parsed_spec["stack"] deployment_type = parsed_spec[constants.deploy_to_key] + + # Branch to Helm chart generation flow early if --helm-chart flag is set + if deployment_type == "k8s" and helm_chart: + from stack_orchestrator.deploy.k8s.helm.chart_generator import generate_helm_chart + generate_helm_chart(stack_name, spec_file, deployment_dir) + return # Exit early, completely separate from existing k8s deployment flow + + # Existing deployment flow continues unchanged stack_file = get_stack_path(stack_name).joinpath(constants.stack_file_name) parsed_stack = get_parsed_stack_config(stack_name) if opts.o.debug: diff --git a/stack_orchestrator/deploy/k8s/helm/__init__.py b/stack_orchestrator/deploy/k8s/helm/__init__.py new file mode 100644 index 00000000..3d935105 --- /dev/null +++ b/stack_orchestrator/deploy/k8s/helm/__init__.py @@ -0,0 +1,14 @@ +# Copyright © 2025 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . diff --git a/stack_orchestrator/deploy/k8s/helm/chart_generator.py b/stack_orchestrator/deploy/k8s/helm/chart_generator.py new file mode 100644 index 00000000..8431bc1d --- /dev/null +++ b/stack_orchestrator/deploy/k8s/helm/chart_generator.py @@ -0,0 +1,266 @@ +# Copyright © 2025 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import shutil +from pathlib import Path + +from stack_orchestrator import constants +from stack_orchestrator.opts import opts +from stack_orchestrator.util import ( + get_stack_path, + get_parsed_stack_config, + get_pod_list, + get_pod_file_path, + error_exit +) +from stack_orchestrator.deploy.k8s.helm.kompose_wrapper import ( + check_kompose_available, + get_kompose_version, + convert_to_helm_chart +) +from stack_orchestrator.util import get_yaml + + +def _post_process_chart(chart_dir: Path, chart_name: str) -> None: + """ + Post-process Kompose-generated chart to fix common issues. + + Fixes: + 1. Chart.yaml name, description and keywords + + TODO: + - Add defaultMode: 0755 to ConfigMap volumes containing scripts (.sh files) + """ + yaml = get_yaml() + + # Fix Chart.yaml + chart_yaml_path = chart_dir / "Chart.yaml" + if chart_yaml_path.exists(): + chart_yaml = yaml.load(open(chart_yaml_path, "r")) + + # Fix name + chart_yaml["name"] = chart_name + + # Fix description + chart_yaml["description"] = f"Generated Helm chart for {chart_name} stack" + + # Fix keywords + if "keywords" in chart_yaml and isinstance(chart_yaml["keywords"], list): + chart_yaml["keywords"] = [chart_name] + + with open(chart_yaml_path, "w") as f: + yaml.dump(chart_yaml, f) + + +def generate_helm_chart(stack_path: str, spec_file: str, deployment_dir: str = None) -> None: + """ + Generate a self-sufficient Helm chart from stack compose files using Kompose. + + Args: + stack_path: Path to the stack directory + spec_file: Path to the deployment spec file + deployment_dir: Optional directory for deployment output + + Output structure: + deployment-dir/ + ├── spec.yml # Reference + ├── stack.yml # Reference + └── chart/ # Self-sufficient Helm chart + ├── Chart.yaml + ├── README.md + └── templates/ + └── *.yaml + + TODO: Enhancements: + - Parse generated templates and extract values to values.yaml + - Replace hardcoded image tags with {{ .Values.image.tag }} + - Replace hardcoded PVC sizes with {{ .Values.persistence.size }} + - Convert Deployments to StatefulSets for stateful services (zenithd, postgres) + - Add _helpers.tpl with common label/selector functions + - Embed config files (scripts, templates) into ConfigMap templates + - Generate Secret templates for validator keys with placeholders + - Add init containers for genesis/config setup + - Enhance Chart.yaml with proper metadata (version, description, etc.) + """ + + parsed_stack = get_parsed_stack_config(stack_path) + stack_name = parsed_stack.get("name", stack_path) + + # 1. Check Kompose availability + if not check_kompose_available(): + error_exit("kompose not found in PATH.\n") + + # 2. Setup deployment directory + if deployment_dir: + deployment_dir_path = Path(deployment_dir) + else: + deployment_dir_path = Path(f"{stack_name}-deployment") + + if deployment_dir_path.exists(): + error_exit(f"Deployment directory already exists: {deployment_dir_path}") + + if opts.o.debug: + print(f"Creating deployment directory: {deployment_dir_path}") + + deployment_dir_path.mkdir(parents=True) + + # 3. Copy spec and stack files to deployment directory (for reference) + spec_path = Path(spec_file).resolve() + if not spec_path.exists(): + error_exit(f"Spec file not found: {spec_file}") + + stack_file_path = get_stack_path(stack_path).joinpath(constants.stack_file_name) + if not stack_file_path.exists(): + error_exit(f"Stack file not found: {stack_file_path}") + + shutil.copy(spec_path, deployment_dir_path / constants.spec_file_name) + shutil.copy(stack_file_path, deployment_dir_path / constants.stack_file_name) + + if opts.o.debug: + print(f"Copied spec file: {spec_path}") + print(f"Copied stack file: {stack_file_path}") + + # 4. Get compose files from stack + pods = get_pod_list(parsed_stack) + if not pods: + error_exit(f"No pods found in stack: {stack_path}") + + # Get clean stack name from stack.yml + chart_name = stack_name.replace("_", "-").replace(" ", "-") + + if opts.o.debug: + print(f"Found {len(pods)} pod(s) in stack: {pods}") + + compose_files = [] + for pod in pods: + pod_file = get_pod_file_path(stack_path, parsed_stack, pod) + if not pod_file.exists(): + error_exit(f"Pod file not found: {pod_file}") + compose_files.append(pod_file) + if opts.o.debug: + print(f"Found compose file: {pod_file.name}") + + try: + version = get_kompose_version() + print(f"Using kompose version: {version}") + except Exception as e: + error_exit(f"Failed to get kompose version: {e}") + + # 5. Create chart directory and invoke Kompose + chart_dir = deployment_dir_path / "chart" + + print(f"Converting {len(compose_files)} compose file(s) to Helm chart using Kompose...") + + try: + output = convert_to_helm_chart( + compose_files=compose_files, + output_dir=chart_dir, + chart_name=chart_name + ) + if opts.o.debug: + print(f"Kompose output:\n{output}") + except Exception as e: + error_exit(f"Helm chart generation failed: {e}") + + # 6. Post-process generated chart + _post_process_chart(chart_dir, chart_name) + + # 7. Generate README.md with basic installation instructions + readme_content = f"""# {chart_name} Helm Chart + +Generated by laconic-so from stack: `{stack_path} + +## Prerequisites + +- Kubernetes cluster (v1.27+) +- Helm (v3.12+) +- kubectl configured to access your cluster + +## Installation + +```bash +# Install the chart +helm install {chart_name} {chart_dir} + +# Check deployment status +kubectl get pods +``` + +## Upgrade + +To apply changes made to chart, perform upgrade: + +```bash +helm upgrade {chart_name} {chart_dir} +``` + +## Uninstallation + +```bash +helm uninstall {chart_name} +``` + +## Configuration + +The chart was generated from Docker Compose files using Kompose. + +### Customization + +Edit the generated template files in `templates/` to customize: +- Image repositories and tags +- Resource limits (CPU, memory) +- Persistent volume sizes +- Replica counts +""" + + readme_path = chart_dir / "README.md" + readme_path.write_text(readme_content) + + if opts.o.debug: + print(f"Generated README: {readme_path}") + + # 7. Success message + print(f"\n{'=' * 60}") + print("✓ Helm chart generated successfully!") + print(f"{'=' * 60}") + print("\nChart details:") + print(f" Name: {chart_name}") + print(f" Location: {chart_dir.absolute()}") + print(f" Stack: {stack_path}") + + # Count generated files + template_files = list((chart_dir / "templates").glob("*.yaml")) if (chart_dir / "templates").exists() else [] + print(f" Files: {len(template_files)} template(s) generated") + + print("\nDeployment directory structure:") + print(f" {deployment_dir_path}/") + print(" ├── spec.yml (reference)") + print(" ├── stack.yml (reference)") + print(" └── chart/ (self-sufficient Helm chart)") + + print("\nNext steps:") + print(" 1. Review the chart:") + print(f" cd {chart_dir}") + print(" cat Chart.yaml") + print("") + print(" 2. Review generated templates:") + print(" ls templates/") + print("") + print(" 3. Install to Kubernetes:") + print(f" helm install {chart_name} {chart_dir}") + print("") + print(" 4. Check deployment:") + print(" kubectl get pods") + print("") diff --git a/stack_orchestrator/deploy/k8s/helm/kompose_wrapper.py b/stack_orchestrator/deploy/k8s/helm/kompose_wrapper.py new file mode 100644 index 00000000..18c3b25c --- /dev/null +++ b/stack_orchestrator/deploy/k8s/helm/kompose_wrapper.py @@ -0,0 +1,109 @@ +# Copyright © 2025 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import subprocess +import shutil +from pathlib import Path +from typing import List + + +def check_kompose_available() -> bool: + """Check if kompose binary is available in PATH.""" + return shutil.which("kompose") is not None + + +def get_kompose_version() -> str: + """ + Get the installed kompose version. + + Returns: + Version string (e.g., "1.34.0") + + Raises: + Exception if kompose is not available + """ + if not check_kompose_available(): + raise Exception("kompose not found in PATH") + + result = subprocess.run( + ["kompose", "version"], + capture_output=True, + text=True, + timeout=10 + ) + + if result.returncode != 0: + raise Exception(f"Failed to get kompose version: {result.stderr}") + + # Parse version from output like "1.34.0 (HEAD)" + # Output format: "1.34.0 (HEAD)" or just "1.34.0" + version_line = result.stdout.strip() + version = version_line.split()[0] if version_line else "unknown" + + return version + + +def convert_to_helm_chart(compose_files: List[Path], output_dir: Path, chart_name: str = None) -> str: + """ + Invoke kompose to convert Docker Compose files to a Helm chart. + + Args: + compose_files: List of paths to docker-compose.yml files + output_dir: Directory where the Helm chart will be generated + chart_name: Optional name for the chart (defaults to directory name) + + Returns: + stdout from kompose command + + Raises: + Exception if kompose conversion fails + """ + if not check_kompose_available(): + raise Exception( + "kompose not found in PATH. " + "Install from: https://kompose.io/installation/" + ) + + # Ensure output directory exists + output_dir.mkdir(parents=True, exist_ok=True) + + # Build kompose command + cmd = ["kompose", "convert"] + + # Add all compose files + for compose_file in compose_files: + if not compose_file.exists(): + raise Exception(f"Compose file not found: {compose_file}") + cmd.extend(["-f", str(compose_file)]) + + # Add chart flag and output directory + cmd.extend(["--chart", "-o", str(output_dir)]) + + # Execute kompose + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=60 + ) + + if result.returncode != 0: + raise Exception( + f"Kompose conversion failed:\n" + f"Command: {' '.join(cmd)}\n" + f"Error: {result.stderr}" + ) + + return result.stdout diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp.py b/stack_orchestrator/deploy/webapp/deploy_webapp.py index 4c91dec3..c51f0781 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp.py @@ -91,6 +91,7 @@ def create_deployment(ctx, deployment_dir, image, url, kube_config, image_regist deploy_command_context, spec_file_name, deployment_dir, + False, None, None )