cerc-io/stack-orchestrator
17
1
Fork 3
Code Issues 112 Pull Requests 19 Actions Packages Projects Releases 480 Wiki Activity

Support for k8s ingress and tls (#659)

Browse Source
This commit is contained in:
David Boreham 2023-11-21 16:04:36 -07:00 committed by GitHub
parent 01029cf7aa
commit 87bedde5cb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 172 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
stack_orchestrator/constants.py
View File

@ -19,6 +19,8 @@ k8s_kind_deploy_type = "k8s-kind"
k8s_deploy_type = "k8s"
kube_config_key = "kube-config"
deploy_to_key = "deploy-to"
network_key = "network"
http_proxy_key = "http-proxy"
image_resigtry_key = "image-registry"
kind_config_filename = "kind-config.yml"
kube_config_filename = "kubeconfig.yml"

6
stack_orchestrator/deploy/deployment_create.py
View File

@ -103,8 +103,8 @@ def _fixup_pod_file(pod, spec, compose_dir):
}
pod["volumes"][volume] = new_volume_spec
# Fix up ports
if "ports" in spec:
spec_ports = spec["ports"]
if "network" in spec and "ports" in spec["network"]:
spec_ports = spec["network"]["ports"]
for container_name, container_ports in spec_ports.items():
if container_name in pod["services"]:
pod["services"][container_name]["ports"] = container_ports
@ -285,7 +285,7 @@ def init(ctx, config, kube_config, image_registry, output, map_ports_to_host):
print(f"Creating spec file for stack: {stack} with content: {spec_file_content}")
ports = _get_mapped_ports(stack, map_ports_to_host)
spec_file_content["ports"] = ports
spec_file_content.update({"network": {"ports": ports}})
named_volumes = _get_named_volumes(stack)
if named_volumes:

83
stack_orchestrator/deploy/k8s/cluster_info.py
View File

@ -22,6 +22,7 @@ from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path
from stack_orchestrator.deploy.k8s.helpers import env_var_map_from_file, envs_from_environment_variables_map
from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment
from stack_orchestrator.deploy.deploy_types import DeployEnvVars
from stack_orchestrator.deploy.spec import Spec
from stack_orchestrator.deploy.images import remote_tag_for_image
@ -29,22 +30,91 @@ class ClusterInfo:
parsed_pod_yaml_map: Any
image_set: Set[str] = set()
app_name: str = "test-app"
deployment_name: str = "test-deployment"
environment_variables: DeployEnvVars
remote_image_repo: str
spec: Spec
def __init__(self) -> None:
pass
def int(self, pod_files: List[str], compose_env_file, remote_image_repo):
def int(self, pod_files: List[str], compose_env_file, spec: Spec):
self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files)
# Find the set of images in the pods
self.image_set = images_for_deployment(pod_files)
self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file))
self.remote_image_repo = remote_image_repo
self.spec = spec
if (opts.o.debug):
print(f"Env vars: {self.environment_variables.map}")
def get_ingress(self):
# No ingress for a deployment that has no http-proxy defined, for now
http_proxy_info_list = self.spec.get_http_proxy()
ingress = None
if http_proxy_info_list:
# TODO: handle multiple definitions
http_proxy_info = http_proxy_info_list[0]
if opts.o.debug:
print(f"http-proxy: {http_proxy_info}")
# TODO: good enough parsing for webapp deployment for now
host_name = http_proxy_info["host-name"]
rules = []
tls = [client.V1IngressTLS(
hosts=[host_name],
secret_name=f"{self.app_name}-tls"
)]
paths = []
for route in http_proxy_info["routes"]:
path = route["path"]
proxy_to = route["proxy-to"]
if opts.o.debug:
print(f"proxy config: {path} -> {proxy_to}")
paths.append(client.V1HTTPIngressPath(
path_type="Prefix",
path=path,
backend=client.V1IngressBackend(
service=client.V1IngressServiceBackend(
# TODO: this looks wrong
name=f"{self.app_name}-service",
# TODO: pull port number from the service
port=client.V1ServiceBackendPort(number=80)
)
)
))
rules.append(client.V1IngressRule(
host=host_name,
http=client.V1HTTPIngressRuleValue(
paths=paths
)
))
spec = client.V1IngressSpec(
tls=tls,
rules=rules
)
ingress = client.V1Ingress(
metadata=client.V1ObjectMeta(
name=f"{self.app_name}-ingress",
annotations={
"kubernetes.io/ingress.class": "nginx",
"cert-manager.io/cluster-issuer": "letsencrypt-prod"
}
),
spec=spec
)
return ingress
def get_service(self):
service = client.V1Service(
metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"),
spec=client.V1ServiceSpec(
type="ClusterIP",
ports=[client.V1ServicePort(
port=80,
target_port=80
)],
selector={"app": self.app_name}
)
)
return service
def get_pvcs(self):
result = []
volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
@ -96,7 +166,8 @@ class ClusterInfo:
service_info = services[service_name]
image = service_info["image"]
# Re-write the image tag for remote deployment
image_to_use = remote_tag_for_image(image, self.remote_image_repo) if self.remote_image_repo is not None else image
image_to_use = remote_tag_for_image(
image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name)
container = client.V1Container(
name=container_name,
@ -123,7 +194,7 @@ class ClusterInfo:
deployment = client.V1Deployment(
api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=self.deployment_name),
metadata=client.V1ObjectMeta(name=f"{self.app_name}-deployment"),
spec=spec,
)
return deployment

92
stack_orchestrator/deploy/k8s/deploy_k8s.py
View File

@ -23,6 +23,15 @@ from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, log_stream
from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo
from stack_orchestrator.opts import opts
from stack_orchestrator.deploy.deployment_context import DeploymentContext
from stack_orchestrator.util import error_exit
def _check_delete_exception(e: client.exceptions.ApiException):
if e.status == 404:
if opts.o.debug:
print("Failed to delete object, continuing")
else:
error_exit(f"k8s api error: {e}")
class K8sDeployer(Deployer):
@ -30,6 +39,7 @@ class K8sDeployer(Deployer):
type: str
core_api: client.CoreV1Api
apps_api: client.AppsV1Api
networking_api: client.NetworkingV1Api
k8s_namespace: str = "default"
kind_cluster_name: str
cluster_info : ClusterInfo
@ -45,7 +55,7 @@ class K8sDeployer(Deployer):
self.deployment_context = deployment_context
self.kind_cluster_name = compose_project_name
self.cluster_info = ClusterInfo()
self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec.obj[constants.image_resigtry_key])
self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec)
if (opts.o.debug):
print(f"Deployment dir: {deployment_context.deployment_dir}")
print(f"Compose files: {compose_files}")
@ -60,9 +70,11 @@ class K8sDeployer(Deployer):
# Get the config file and pass to load_kube_config()
config.load_kube_config(config_file=self.deployment_dir.joinpath(constants.kube_config_filename).as_posix())
self.core_api = client.CoreV1Api()
self.networking_api = client.NetworkingV1Api()
self.apps_api = client.AppsV1Api()
def up(self, detach, services):
if self.is_kind():
# Create the kind cluster
create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath(constants.kind_config_filename))
@ -102,6 +114,26 @@ class K8sDeployer(Deployer):
print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \
{deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}")
service: client.V1Service = self.cluster_info.get_service()
service_resp = self.core_api.create_namespaced_service(
namespace=self.k8s_namespace,
body=service
)
if opts.o.debug:
print("Service created:")
print(f"{service_resp}")
# TODO: disable ingress for kind
ingress: client.V1Ingress = self.cluster_info.get_ingress()
ingress_resp = self.networking_api.create_namespaced_ingress(
namespace=self.k8s_namespace,
body=ingress
)
if opts.o.debug:
print("Ingress created:")
print(f"{ingress_resp}")
def down(self, timeout, volumes):
self.connect_api()
# Delete the k8s objects
@ -110,28 +142,60 @@ class K8sDeployer(Deployer):
for pv in pvs:
if opts.o.debug:
print(f"Deleting this pv: {pv}")
pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name)
if opts.o.debug:
print("PV deleted:")
print(f"{pv_resp}")
try:
pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name)
if opts.o.debug:
print("PV deleted:")
print(f"{pv_resp}")
except client.exceptions.ApiException as e:
_check_delete_exception(e)
# Figure out the PVCs for this deployment
pvcs = self.cluster_info.get_pvcs()
for pvc in pvcs:
if opts.o.debug:
print(f"Deleting this pvc: {pvc}")
pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim(name=pvc.metadata.name, namespace=self.k8s_namespace)
if opts.o.debug:
print("PVCs deleted:")
print(f"{pvc_resp}")
# Process compose files into a Deployment
try:
pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim(
name=pvc.metadata.name, namespace=self.k8s_namespace
)
if opts.o.debug:
print("PVCs deleted:")
print(f"{pvc_resp}")
except client.exceptions.ApiException as e:
_check_delete_exception(e)
deployment = self.cluster_info.get_deployment()
# Create the k8s objects
if opts.o.debug:
print(f"Deleting this deployment: {deployment}")
self.apps_api.delete_namespaced_deployment(
name=deployment.metadata.name, namespace=self.k8s_namespace
)
try:
self.apps_api.delete_namespaced_deployment(
name=deployment.metadata.name, namespace=self.k8s_namespace
)
except client.exceptions.ApiException as e:
_check_delete_exception(e)
service: client.V1Service = self.cluster_info.get_service()
if opts.o.debug:
print(f"Deleting service: {service}")
try:
self.core_api.delete_namespaced_service(
namespace=self.k8s_namespace,
name=service.metadata.name
)
except client.exceptions.ApiException as e:
_check_delete_exception(e)
# TODO: disable ingress for kind
ingress: client.V1Ingress = self.cluster_info.get_ingress()
if opts.o.debug:
print(f"Deleting this ingress: {ingress}")
try:
self.networking_api.delete_namespaced_ingress(
name=ingress.metadata.name, namespace=self.k8s_namespace
)
except client.exceptions.ApiException as e:
_check_delete_exception(e)
if self.is_kind():
# Destroy the kind cluster
destroy_cluster(self.kind_cluster_name)

12
stack_orchestrator/deploy/spec.py
View File

@ -16,6 +16,7 @@
from pathlib import Path
import typing
from stack_orchestrator.util import get_yaml
from stack_orchestrator import constants
class Spec:
@ -28,3 +29,14 @@ class Spec:
def init_from_file(self, file_path: Path):
with file_path:
self.obj = get_yaml().load(open(file_path, "r"))
def get_image_registry(self):
return (self.obj[constants.image_resigtry_key]
if self.obj and constants.image_resigtry_key in self.obj
else None)
def get_http_proxy(self):
return (self.obj[constants.network_key][constants.http_proxy_key]
if self.obj and constants.network_key in self.obj
and constants.http_proxy_key in self.obj[constants.network_key]
else None)