diff --git a/HELM_CHART_GENERATION.md b/HELM_CHART_GENERATION.md new file mode 100644 index 00000000..f9fd07c4 --- /dev/null +++ b/HELM_CHART_GENERATION.md @@ -0,0 +1,80 @@ +# Helm Chart Generation + +Generate Kubernetes Helm charts from stack compose files using Kompose. + +## Prerequisites + +Install Kompose: + +```bash +# Linux +curl -L https://github.com/kubernetes/kompose/releases/download/v1.34.0/kompose-linux-amd64 -o kompose +chmod +x kompose +sudo mv kompose /usr/local/bin/ + +# macOS +brew install kompose + +# Verify +kompose version +``` + +## Usage + +### 1. Create spec file + +```bash +laconic-so --stack deploy init \ + --deploy-to k8s \ + --kube-config ~/.kube/config \ + --output spec.yml +``` + +### 2. Generate Helm chart + +```bash +laconic-so --stack deploy create \ + --spec-file spec.yml \ + --deployment-dir my-deployment \ + --helm-chart +``` + +### 3. Deploy to Kubernetes + +```bash +cd my-deployment/chart +helm install my-release ./ --namespace zenith --create-namespace +kubectl get pods -n zenith +``` + +## Output Structure + +```bash +my-deployment/ +├── spec.yml # Reference +├── stack.yml # Reference +└── chart/ # Helm chart + ├── Chart.yaml + ├── README.md + └── templates/ + └── *.yaml +``` + +## Example + +```bash +# Generate chart for stage1-zenithd +laconic-so --stack stage1-zenithd deploy init \ + --deploy-to k8s \ + --kube-config ~/.kube/config \ + --output stage1-spec.yml + +laconic-so --stack stage1-zenithd deploy create \ + --spec-file stage1-spec.yml \ + --deployment-dir stage1-deployment \ + --helm-chart + +# Deploy +cd stage1-deployment/chart +helm install stage1-zenithd ./ --namespace zenith --create-namespace +``` diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index 9d45f226..0b3a92f7 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -443,22 +443,31 @@ def _check_volume_definitions(spec): @click.command() @click.option("--spec-file", required=True, help="Spec file to use to create this deployment") @click.option("--deployment-dir", help="Create deployment files in this directory") +@click.option("--helm-chart", is_flag=True, default=False, help="Generate Helm chart instead of deploying (k8s only)") # TODO: Hack @click.option("--network-dir", help="Network configuration supplied in this directory") @click.option("--initial-peers", help="Initial set of persistent peers") @click.pass_context -def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): +def create(ctx, spec_file, deployment_dir, helm_chart, network_dir, initial_peers): deployment_command_context = ctx.obj - return create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers) + return create_operation(deployment_command_context, spec_file, deployment_dir, helm_chart, network_dir, initial_peers) # The init command's implementation is in a separate function so that we can # call it from other commands, bypassing the click decoration stuff -def create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers): +def create_operation(deployment_command_context, spec_file, deployment_dir, helm_chart, network_dir, initial_peers): parsed_spec = Spec(os.path.abspath(spec_file), get_parsed_deployment_spec(spec_file)) _check_volume_definitions(parsed_spec) stack_name = parsed_spec["stack"] deployment_type = parsed_spec[constants.deploy_to_key] + + # Branch to Helm chart generation flow early if --helm-chart flag is set + if deployment_type == "k8s" and helm_chart: + from stack_orchestrator.deploy.k8s.helm.chart_generator import generate_helm_chart + generate_helm_chart(stack_name, spec_file, deployment_dir) + return # Exit early, completely separate from existing k8s deployment flow + + # Existing deployment flow continues unchanged stack_file = get_stack_path(stack_name).joinpath(constants.stack_file_name) parsed_stack = get_parsed_stack_config(stack_name) if opts.o.debug: diff --git a/stack_orchestrator/deploy/k8s/helm/__init__.py b/stack_orchestrator/deploy/k8s/helm/__init__.py new file mode 100644 index 00000000..3d935105 --- /dev/null +++ b/stack_orchestrator/deploy/k8s/helm/__init__.py @@ -0,0 +1,14 @@ +# Copyright © 2025 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . diff --git a/stack_orchestrator/deploy/k8s/helm/chart_generator.py b/stack_orchestrator/deploy/k8s/helm/chart_generator.py new file mode 100644 index 00000000..eacb5120 --- /dev/null +++ b/stack_orchestrator/deploy/k8s/helm/chart_generator.py @@ -0,0 +1,266 @@ +# Copyright © 2025 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import shutil +from pathlib import Path + +from stack_orchestrator import constants +from stack_orchestrator.opts import opts +from stack_orchestrator.util import ( + get_stack_path, + get_parsed_stack_config, + get_pod_list, + get_pod_file_path, + error_exit +) +from stack_orchestrator.deploy.k8s.helm.kompose_wrapper import ( + check_kompose_available, + get_kompose_version, + convert_to_helm_chart +) + + +def generate_helm_chart(stack_name: str, spec_file: str, deployment_dir: str = None) -> None: + """ + Generate a self-sufficient Helm chart from stack compose files using Kompose. + + This is completely separate from the existing k8s deployment flow. + + Args: + stack_name: Name of the stack + spec_file: Path to the deployment spec file + deployment_dir: Optional directory for deployment output + + Output structure: + deployment-dir/ + ├── spec.yml # Reference + ├── stack.yml # Reference + └── chart/ # Self-sufficient Helm chart + ├── Chart.yaml + ├── values.yaml (generated by Kompose) + ├── README.md + └── templates/ + └── *.yaml + + TODO: Post-processing enhancements: + - Parse generated templates and extract values to values.yaml + - Replace hardcoded image tags with {{ .Values.image.tag }} + - Replace hardcoded replica counts with {{ .Values.replicaCount }} + - Replace hardcoded PVC sizes with {{ .Values.persistence.size }} + - Convert Deployments to StatefulSets for stateful services (zenithd, postgres) + - Add _helpers.tpl with common label/selector functions + - Embed genesis.json into ConfigMap template + - Embed config files (scripts, templates) into ConfigMap templates + - Generate Secret templates for validator keys with placeholders + - Add init containers for genesis/config setup + - Enhance Chart.yaml with proper metadata (version, description, etc.) + - Generate comprehensive values.yaml documentation + """ + + # 1. Setup deployment directory + if deployment_dir: + deployment_dir_path = Path(deployment_dir) + else: + deployment_dir_path = Path("deployment-001") + + if deployment_dir_path.exists(): + error_exit(f"Deployment directory already exists: {deployment_dir_path}") + + if opts.o.debug: + print(f"Creating deployment directory: {deployment_dir_path}") + + deployment_dir_path.mkdir(parents=True) + + # 2. Copy spec and stack files to deployment directory (for reference) + spec_path = Path(spec_file).resolve() + if not spec_path.exists(): + error_exit(f"Spec file not found: {spec_file}") + + stack_file_path = get_stack_path(stack_name).joinpath(constants.stack_file_name) + if not stack_file_path.exists(): + error_exit(f"Stack file not found: {stack_file_path}") + + shutil.copy(spec_path, deployment_dir_path / constants.spec_file_name) + shutil.copy(stack_file_path, deployment_dir_path / constants.stack_file_name) + + if opts.o.debug: + print(f"Copied spec file: {spec_path}") + print(f"Copied stack file: {stack_file_path}") + + # 3. Get compose files from stack + parsed_stack = get_parsed_stack_config(stack_name) + pods = get_pod_list(parsed_stack) + + if not pods: + error_exit(f"No pods found in stack: {stack_name}") + + if opts.o.debug: + print(f"Found {len(pods)} pod(s) in stack: {pods}") + + compose_files = [] + for pod in pods: + pod_file = get_pod_file_path(stack_name, parsed_stack, pod) + if not pod_file.exists(): + error_exit(f"Pod file not found: {pod_file}") + compose_files.append(pod_file) + if opts.o.debug: + print(f"Found compose file: {pod_file.name}") + + # 4. Check Kompose availability + if not check_kompose_available(): + error_exit( + "kompose not found in PATH.\n" + "Install from: https://kompose.io/installation/\n" + " - Linux: curl -L https://github.com/kubernetes/kompose/releases/download/v1.34.0/kompose-linux-amd64 -o kompose\n" + " - macOS: brew install kompose" + ) + + try: + version = get_kompose_version() + print(f"Using kompose version: {version}") + except Exception as e: + error_exit(f"Failed to get kompose version: {e}") + + # 5. Create chart directory and invoke Kompose + chart_dir = deployment_dir_path / "chart" + chart_name = stack_name.replace("_", "-").replace(" ", "-") # Helm chart names prefer dashes + + print(f"Converting {len(compose_files)} compose file(s) to Helm chart using Kompose...") + + try: + output = convert_to_helm_chart( + compose_files=compose_files, + output_dir=chart_dir, + chart_name=chart_name + ) + if opts.o.debug: + print(f"Kompose output:\n{output}") + except Exception as e: + error_exit(f"Helm chart generation failed: {e}") + + # 6. Generate README.md with basic installation instructions + readme_content = f"""# {chart_name} Helm Chart + +Generated by laconic-so from stack: {stack_name} + +## Prerequisites + +- Kubernetes cluster (v1.27+) +- Helm (v3.12+) +- kubectl configured to access your cluster + +## Installation + +```bash +# Create namespace (optional) +kubectl create namespace zenith + +# Install the chart +helm install {chart_name} ./ --namespace zenith + +# Check deployment status +kubectl get pods -n zenith +``` + +## Uninstallation + +```bash +helm uninstall {chart_name} --namespace zenith +``` + +## Configuration + +The chart was generated from Docker Compose files using Kompose. + +### Customization + +Edit the generated template files in `templates/` to customize: +- Image repositories and tags +- Resource limits (CPU, memory) +- Persistent volume sizes +- Replica counts + +TODO: values.yaml will be enhanced in future versions to provide a cleaner configuration interface. + +## Generated Files + +- `Chart.yaml` - Chart metadata +- `templates/` - Kubernetes resource manifests + - Deployments + - Services + - PersistentVolumeClaims + - ConfigMaps + +## Post-Processing TODO + +The following enhancements are planned: +- Parameterized values.yaml with common configuration options +- StatefulSet conversion for stateful services (databases, validators) +- Embedded deployment artifacts (genesis files, configs) as ConfigMaps +- Secret templates for sensitive data (validator keys, passwords) +- Init containers for setup and configuration +- Enhanced Chart.yaml metadata + +## Notes + +This chart is generated from compose files and may require manual adjustments for production use: +1. Review resource limits in deployment manifests +2. Configure persistent volume storage classes for your cluster +3. Set up ingress for external access if needed +4. Configure secrets for sensitive data (passwords, keys) +5. Adjust service types (ClusterIP, NodePort, LoadBalancer) based on your needs + +For more information, see the laconic-so documentation. +""" + + readme_path = chart_dir / "README.md" + readme_path.write_text(readme_content) + + if opts.o.debug: + print(f"Generated README: {readme_path}") + + # 7. Success message + print(f"\n{'=' * 60}") + print(f"✓ Helm chart generated successfully!") + print(f"{'=' * 60}") + print(f"\nChart details:") + print(f" Name: {chart_name}") + print(f" Location: {chart_dir.absolute()}") + print(f" Stack: {stack_name}") + + # Count generated files + template_files = list((chart_dir / "templates").glob("*.yaml")) if (chart_dir / "templates").exists() else [] + print(f" Files: {len(template_files)} template(s) generated") + + print(f"\nDeployment directory structure:") + print(f" {deployment_dir_path}/") + print(f" ├── spec.yml (reference)") + print(f" ├── stack.yml (reference)") + print(f" └── chart/ (self-sufficient Helm chart)") + + print(f"\nNext steps:") + print(f" 1. Review the chart:") + print(f" cd {chart_dir}") + print(f" cat Chart.yaml") + print(f"") + print(f" 2. Review generated templates:") + print(f" ls templates/") + print(f"") + print(f" 3. Install to Kubernetes:") + print(f" helm install {chart_name} ./ --namespace zenith --create-namespace") + print(f"") + print(f" 4. Check deployment:") + print(f" kubectl get pods -n zenith") + print(f"") diff --git a/stack_orchestrator/deploy/k8s/helm/kompose_wrapper.py b/stack_orchestrator/deploy/k8s/helm/kompose_wrapper.py new file mode 100644 index 00000000..18c3b25c --- /dev/null +++ b/stack_orchestrator/deploy/k8s/helm/kompose_wrapper.py @@ -0,0 +1,109 @@ +# Copyright © 2025 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import subprocess +import shutil +from pathlib import Path +from typing import List + + +def check_kompose_available() -> bool: + """Check if kompose binary is available in PATH.""" + return shutil.which("kompose") is not None + + +def get_kompose_version() -> str: + """ + Get the installed kompose version. + + Returns: + Version string (e.g., "1.34.0") + + Raises: + Exception if kompose is not available + """ + if not check_kompose_available(): + raise Exception("kompose not found in PATH") + + result = subprocess.run( + ["kompose", "version"], + capture_output=True, + text=True, + timeout=10 + ) + + if result.returncode != 0: + raise Exception(f"Failed to get kompose version: {result.stderr}") + + # Parse version from output like "1.34.0 (HEAD)" + # Output format: "1.34.0 (HEAD)" or just "1.34.0" + version_line = result.stdout.strip() + version = version_line.split()[0] if version_line else "unknown" + + return version + + +def convert_to_helm_chart(compose_files: List[Path], output_dir: Path, chart_name: str = None) -> str: + """ + Invoke kompose to convert Docker Compose files to a Helm chart. + + Args: + compose_files: List of paths to docker-compose.yml files + output_dir: Directory where the Helm chart will be generated + chart_name: Optional name for the chart (defaults to directory name) + + Returns: + stdout from kompose command + + Raises: + Exception if kompose conversion fails + """ + if not check_kompose_available(): + raise Exception( + "kompose not found in PATH. " + "Install from: https://kompose.io/installation/" + ) + + # Ensure output directory exists + output_dir.mkdir(parents=True, exist_ok=True) + + # Build kompose command + cmd = ["kompose", "convert"] + + # Add all compose files + for compose_file in compose_files: + if not compose_file.exists(): + raise Exception(f"Compose file not found: {compose_file}") + cmd.extend(["-f", str(compose_file)]) + + # Add chart flag and output directory + cmd.extend(["--chart", "-o", str(output_dir)]) + + # Execute kompose + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=60 + ) + + if result.returncode != 0: + raise Exception( + f"Kompose conversion failed:\n" + f"Command: {' '.join(cmd)}\n" + f"Error: {result.stderr}" + ) + + return result.stdout