From 8964e1c0fea2bbda79a0bc9457d6076d4778dc13 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 7 Feb 2024 16:48:02 -0600 Subject: [PATCH 1/4] Add resource limit options to spec. --- stack_orchestrator/deploy/k8s/cluster_info.py | 28 ++++++++++++++++--- stack_orchestrator/deploy/spec.py | 6 ++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 35b2b9da..527fe7c5 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -28,6 +28,15 @@ from stack_orchestrator.deploy.deploy_types import DeployEnvVars from stack_orchestrator.deploy.spec import Spec from stack_orchestrator.deploy.images import remote_tag_for_image +DEFAULT_VOLUME_RESOURCES = { + "requests": {"storage": "2Gi"} +} + +DEFAULT_CONTAINER_RESOURCES = { + "requests": {"cpu": "100m", "memory": "200Mi"}, + "limits": {"cpu": "1000m", "memory": "2000Mi"}, +} + class ClusterInfo: parsed_pod_yaml_map: Any @@ -135,6 +144,9 @@ class ClusterInfo: result = [] spec_volumes = self.spec.get_volumes() named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) + resources = self.spec.get_volume_resources() + if not resources: + resources = DEFAULT_VOLUME_RESOURCES if opts.o.debug: print(f"Spec Volumes: {spec_volumes}") print(f"Named Volumes: {named_volumes}") @@ -147,7 +159,8 @@ class ClusterInfo: access_modes=["ReadWriteOnce"], storage_class_name="manual", resources=client.V1ResourceRequirements( - requests={"storage": "2Gi"} + requests=resources.get("requests"), + limits=resources.get("limits") ), volume_name=f"{self.app_name}-{volume_name}" ) @@ -192,6 +205,9 @@ class ClusterInfo: result = [] spec_volumes = self.spec.get_volumes() named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) + resources = self.spec.get_volume_resources() + if not resources: + resources = DEFAULT_VOLUME_RESOURCES for volume_name in spec_volumes: if volume_name not in named_volumes: if opts.o.debug: @@ -200,7 +216,7 @@ class ClusterInfo: spec = client.V1PersistentVolumeSpec( storage_class_name="manual", access_modes=["ReadWriteOnce"], - capacity={"storage": "2Gi"}, + capacity=resources.get("requests", DEFAULT_VOLUME_RESOURCES["requests"]), host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name)) ) pv = client.V1PersistentVolume( @@ -214,6 +230,10 @@ class ClusterInfo: # TODO: put things like image pull policy into an object-scope struct def get_deployment(self, image_pull_policy: str = None): containers = [] + resources = self.spec.get_container_resources() + if not resources: + resources = DEFAULT_CONTAINER_RESOURCES + print(resources) for pod_name in self.parsed_pod_yaml_map: pod = self.parsed_pod_yaml_map[pod_name] services = pod["services"] @@ -238,8 +258,8 @@ class ClusterInfo: ports=[client.V1ContainerPort(container_port=port)], volume_mounts=volume_mounts, resources=client.V1ResourceRequirements( - requests={"cpu": "100m", "memory": "200Mi"}, - limits={"cpu": "1000m", "memory": "2000Mi"}, + requests=resources.get("requests"), + limits=resources.get("limits") ), ) containers.append(container) diff --git a/stack_orchestrator/deploy/spec.py b/stack_orchestrator/deploy/spec.py index dd6cd107..e7fa8583 100644 --- a/stack_orchestrator/deploy/spec.py +++ b/stack_orchestrator/deploy/spec.py @@ -47,6 +47,12 @@ class Spec: if self.obj and "configmaps" in self.obj else {}) + def get_container_resources(self): + return self.obj.get("resources", {}).get("containers") + + def get_volume_resources(self): + return self.obj.get("resources", {}).get("volumes") + def get_http_proxy(self): return (self.obj[constants.network_key][constants.http_proxy_key] if self.obj and constants.network_key in self.obj From 2a9955055ce4452c0eba725036e3ec6c2c122043 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 7 Feb 2024 16:56:35 -0600 Subject: [PATCH 2/4] debug --- stack_orchestrator/deploy/k8s/cluster_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 527fe7c5..c65a299d 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -150,6 +150,7 @@ class ClusterInfo: if opts.o.debug: print(f"Spec Volumes: {spec_volumes}") print(f"Named Volumes: {named_volumes}") + print(f"Resources: {resources}") for volume_name in spec_volumes: if volume_name not in named_volumes: if opts.o.debug: @@ -233,7 +234,6 @@ class ClusterInfo: resources = self.spec.get_container_resources() if not resources: resources = DEFAULT_CONTAINER_RESOURCES - print(resources) for pod_name in self.parsed_pod_yaml_map: pod = self.parsed_pod_yaml_map[pod_name] services = pod["services"] From 4b3b3478e7b931aec88e6509e2acdba7b44fc358 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 8 Feb 2024 00:43:41 -0600 Subject: [PATCH 3/4] Switch to Docker-style limits --- requirements.txt | 1 + stack_orchestrator/deploy/k8s/cluster_info.py | 48 +++++++++++------ stack_orchestrator/deploy/spec.py | 54 +++++++++++++++++-- 3 files changed, 83 insertions(+), 20 deletions(-) diff --git a/requirements.txt b/requirements.txt index bbf97b4a..f6e3d07c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ pydantic==1.10.9 tomli==2.0.1 validators==0.22.0 kubernetes>=28.1.0 +humanfriendly>=10.0 diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index c65a299d..6f7d0cec 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -25,17 +25,37 @@ from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment from stack_orchestrator.deploy.deploy_types import DeployEnvVars -from stack_orchestrator.deploy.spec import Spec +from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits from stack_orchestrator.deploy.images import remote_tag_for_image -DEFAULT_VOLUME_RESOURCES = { - "requests": {"storage": "2Gi"} -} +DEFAULT_VOLUME_RESOURCES = Resources({ + "reservations": {"storage": "2Gi"} +}) -DEFAULT_CONTAINER_RESOURCES = { - "requests": {"cpu": "100m", "memory": "200Mi"}, - "limits": {"cpu": "1000m", "memory": "2000Mi"}, -} +DEFAULT_CONTAINER_RESOURCES = Resources({ + "reservations": {"cpus": "0.1", "memory": "200M"}, + "limits": {"cpus": "1.0", "memory": "2000M"}, +}) + + +def get_k8s_resource_requirements(resources: Resources) -> client.V1ResourceRequirements: + def to_dict(limits: ResourceLimits): + if not limits: + return None + + ret = {} + if limits.cpus: + ret["cpu"] = str(limits.cpus) + if limits.memory: + ret["memory"] = f"{int(limits.memory / (1000 * 1000))}M" + if limits.storage: + ret["storage"] = f"{int(limits.storage / (1000 * 1000))}M" + return ret + + return client.V1ResourceRequirements( + requests=to_dict(resources.reservations), + limits=to_dict(resources.limits) + ) class ClusterInfo: @@ -159,10 +179,7 @@ class ClusterInfo: spec = client.V1PersistentVolumeClaimSpec( access_modes=["ReadWriteOnce"], storage_class_name="manual", - resources=client.V1ResourceRequirements( - requests=resources.get("requests"), - limits=resources.get("limits") - ), + resources=get_k8s_resource_requirements(resources), volume_name=f"{self.app_name}-{volume_name}" ) pvc = client.V1PersistentVolumeClaim( @@ -217,7 +234,7 @@ class ClusterInfo: spec = client.V1PersistentVolumeSpec( storage_class_name="manual", access_modes=["ReadWriteOnce"], - capacity=resources.get("requests", DEFAULT_VOLUME_RESOURCES["requests"]), + capacity=get_k8s_resource_requirements(resources).requests, host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name)) ) pv = client.V1PersistentVolume( @@ -257,10 +274,7 @@ class ClusterInfo: env=envs_from_environment_variables_map(self.environment_variables.map), ports=[client.V1ContainerPort(container_port=port)], volume_mounts=volume_mounts, - resources=client.V1ResourceRequirements( - requests=resources.get("requests"), - limits=resources.get("limits") - ), + resources=get_k8s_resource_requirements(resources), ) containers.append(container) volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec, self.app_name) diff --git a/stack_orchestrator/deploy/spec.py b/stack_orchestrator/deploy/spec.py index e7fa8583..fa0489e7 100644 --- a/stack_orchestrator/deploy/spec.py +++ b/stack_orchestrator/deploy/spec.py @@ -13,12 +13,60 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from pathlib import Path import typing +import humanfriendly + +from pathlib import Path + from stack_orchestrator.util import get_yaml from stack_orchestrator import constants +class ResourceLimits: + cpus: float = None + memory: int = None + storage: int = None + + def __init__(self, obj={}): + if "cpus" in obj: + self.cpus = float(obj["cpus"]) + if "memory" in obj: + self.memory = humanfriendly.parse_size(obj["memory"]) + if "storage" in obj: + self.storage = humanfriendly.parse_size(obj["storage"]) + + def __len__(self): + return len(self.__dict__) + + def __iter__(self): + for k in self.__dict__: + yield k, self.__dict__[k] + + def __repr__(self): + return str(self.__dict__) + + +class Resources: + limits: ResourceLimits = None + reservations: ResourceLimits = None + + def __init__(self, obj={}): + if "reservations" in obj: + self.reservations = ResourceLimits(obj["reservations"]) + if "limits" in obj: + self.limits = ResourceLimits(obj["limits"]) + + def __len__(self): + return len(self.__dict__) + + def __iter__(self): + for k in self.__dict__: + yield k, self.__dict__[k] + + def __repr__(self): + return str(self.__dict__) + + class Spec: obj: typing.Any @@ -48,10 +96,10 @@ class Spec: else {}) def get_container_resources(self): - return self.obj.get("resources", {}).get("containers") + return Resources(self.obj.get("resources", {}).get("containers", {})) def get_volume_resources(self): - return self.obj.get("resources", {}).get("volumes") + return Resources(self.obj.get("resources", {}).get("volumes", {})) def get_http_proxy(self): return (self.obj[constants.network_key][constants.http_proxy_key] From 3309782439373ab252639c0010ae9369de4a8885 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 8 Feb 2024 00:47:46 -0600 Subject: [PATCH 4/4] Refactor --- stack_orchestrator/deploy/k8s/cluster_info.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 6f7d0cec..30b2ab11 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -38,7 +38,7 @@ DEFAULT_CONTAINER_RESOURCES = Resources({ }) -def get_k8s_resource_requirements(resources: Resources) -> client.V1ResourceRequirements: +def to_k8s_resource_requirements(resources: Resources) -> client.V1ResourceRequirements: def to_dict(limits: ResourceLimits): if not limits: return None @@ -179,7 +179,7 @@ class ClusterInfo: spec = client.V1PersistentVolumeClaimSpec( access_modes=["ReadWriteOnce"], storage_class_name="manual", - resources=get_k8s_resource_requirements(resources), + resources=to_k8s_resource_requirements(resources), volume_name=f"{self.app_name}-{volume_name}" ) pvc = client.V1PersistentVolumeClaim( @@ -234,7 +234,7 @@ class ClusterInfo: spec = client.V1PersistentVolumeSpec( storage_class_name="manual", access_modes=["ReadWriteOnce"], - capacity=get_k8s_resource_requirements(resources).requests, + capacity=to_k8s_resource_requirements(resources).requests, host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name)) ) pv = client.V1PersistentVolume( @@ -274,7 +274,7 @@ class ClusterInfo: env=envs_from_environment_variables_map(self.environment_variables.map), ports=[client.V1ContainerPort(container_port=port)], volume_mounts=volume_mounts, - resources=get_k8s_resource_requirements(resources), + resources=to_k8s_resource_requirements(resources), ) containers.append(container) volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec, self.app_name)