From aa0f60baa1bb5669cdf653b9a442430f54cf94a6 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 25 Jan 2024 06:46:18 -0700 Subject: [PATCH 01/27] Trigger fixturenet-eth-plugeth-test run --- .gitea/workflows/triggers/fixturenet-eth-plugeth-test | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitea/workflows/triggers/fixturenet-eth-plugeth-test b/.gitea/workflows/triggers/fixturenet-eth-plugeth-test index e0c6da50..ca2c643e 100644 --- a/.gitea/workflows/triggers/fixturenet-eth-plugeth-test +++ b/.gitea/workflows/triggers/fixturenet-eth-plugeth-test @@ -1,2 +1,3 @@ Change this file to trigger running the fixturenet-eth-plugeth-test CI job trigger +trigger From 0a302ea555cc6c9a344fa1067cf5d572a0b2a85e Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 25 Jan 2024 07:33:44 -0700 Subject: [PATCH 02/27] Add a schedule for fixturenet-eth-plugeth-test --- .gitea/workflows/fixturenet-eth-plugeth-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitea/workflows/fixturenet-eth-plugeth-test.yml b/.gitea/workflows/fixturenet-eth-plugeth-test.yml index 739e33b0..11bca169 100644 --- a/.gitea/workflows/fixturenet-eth-plugeth-test.yml +++ b/.gitea/workflows/fixturenet-eth-plugeth-test.yml @@ -6,6 +6,8 @@ on: paths: - '!**' - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test' + schedule: # Note: coordinate with other tests to not overload runners at the same time of day + - cron: '2 14 * * *' # Needed until we can incorporate docker startup into the executor container env: From b9128841e4caf90b29cf50979d650a778127bd13 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 25 Jan 2024 11:42:07 -0700 Subject: [PATCH 03/27] Switch test back to stock runner --- .gitea/workflows/test-k8s-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/test-k8s-deploy.yml b/.gitea/workflows/test-k8s-deploy.yml index edd87177..3c482d81 100644 --- a/.gitea/workflows/test-k8s-deploy.yml +++ b/.gitea/workflows/test-k8s-deploy.yml @@ -10,7 +10,7 @@ on: jobs: test: name: "Run deploy test suite on kind/k8s" - runs-on: ubuntu-22.04-with-syn-ethdb + runs-on: ubuntu-22.04 steps: - name: "Clone project repository" uses: actions/checkout@v3 From eca52b10b7a2a0725588b6ebdbd162c758a06b61 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 25 Jan 2024 11:46:50 -0700 Subject: [PATCH 04/27] Fix copy-paste error --- .gitea/workflows/test-k8s-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/test-k8s-deploy.yml b/.gitea/workflows/test-k8s-deploy.yml index 3c482d81..b9463aa7 100644 --- a/.gitea/workflows/test-k8s-deploy.yml +++ b/.gitea/workflows/test-k8s-deploy.yml @@ -5,7 +5,7 @@ on: branches: '*' paths: - '!**' - - '.gitea/workflows/triggers/fixturenet-laconicd-test' + - '.gitea/workflows/triggers/test-k8s-deploy' jobs: test: From b7f215d9bfba3e6bd1b9d8b3068077e213500180 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Sun, 28 Jan 2024 16:21:39 -0700 Subject: [PATCH 05/27] k8s test fixes (#713) * Add cgroup setup, increase test timeouts * Trigger from test script or CI job changes too --- .gitea/workflows/test-k8s-deploy.yml | 7 ++++++- tests/k8s-deploy/run-deploy-test.sh | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitea/workflows/test-k8s-deploy.yml b/.gitea/workflows/test-k8s-deploy.yml index b9463aa7..208b8b0c 100644 --- a/.gitea/workflows/test-k8s-deploy.yml +++ b/.gitea/workflows/test-k8s-deploy.yml @@ -6,6 +6,8 @@ on: paths: - '!**' - '.gitea/workflows/triggers/test-k8s-deploy' + - '.gitea/workflows/test-k8s-deploy.yml' + - 'tests/k8s-deploy/run-deploy-test.sh' jobs: test: @@ -39,4 +41,7 @@ jobs: - name: "Install Kubectl" run: ./tests/scripts/install-kubectl.sh - name: "Run k8s deployment test" - run: ./tests/k8s-deploy/run-deploy-test.sh + run: | + source /opt/bash-utils/cgroup-helper.sh + join_cgroup + ./tests/k8s-deploy/run-deploy-test.sh diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh index 15eb2d3b..0c7246e0 100755 --- a/tests/k8s-deploy/run-deploy-test.sh +++ b/tests/k8s-deploy/run-deploy-test.sh @@ -9,7 +9,7 @@ fi # Helper functions: TODO move into a separate file wait_for_pods_started () { - for i in {1..5} + for i in {1..50} do local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps ) @@ -27,7 +27,7 @@ wait_for_pods_started () { } wait_for_log_output () { - for i in {1..5} + for i in {1..50} do local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) From a750b645b90259f3148c44f92c80f615dc18ce3d Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 30 Jan 2024 11:18:08 -0700 Subject: [PATCH 06/27] Merge Ci test branch fixes (#717) --- .gitea/workflows/test-k8s-deploy.yml | 3 +++ tests/k8s-deploy/run-deploy-test.sh | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/test-k8s-deploy.yml b/.gitea/workflows/test-k8s-deploy.yml index 208b8b0c..92c76a7d 100644 --- a/.gitea/workflows/test-k8s-deploy.yml +++ b/.gitea/workflows/test-k8s-deploy.yml @@ -36,6 +36,8 @@ jobs: run: ./scripts/create_build_tag_file.sh - name: "Build local shiv package" run: ./scripts/build_shiv_package.sh + - name: "Check cgroups version" + run: mount | grep cgroup - name: "Install kind" run: ./tests/scripts/install-kind.sh - name: "Install Kubectl" @@ -45,3 +47,4 @@ jobs: source /opt/bash-utils/cgroup-helper.sh join_cgroup ./tests/k8s-deploy/run-deploy-test.sh + diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh index 0c7246e0..de1a5f10 100755 --- a/tests/k8s-deploy/run-deploy-test.sh +++ b/tests/k8s-deploy/run-deploy-test.sh @@ -120,7 +120,8 @@ fi # Stop then start again and check the volume was preserved $TEST_TARGET_SO deployment --dir $test_deployment_dir stop # Sleep a bit just in case -sleep 2 +# sleep for longer to check if that's why the subsequent create cluster fails +sleep 20 $TEST_TARGET_SO deployment --dir $test_deployment_dir start wait_for_pods_started wait_for_log_output From 428b05158ed44affe1668e5f420972269dd53f6f Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 30 Jan 2024 13:31:59 -0600 Subject: [PATCH 07/27] Fix DnsRecord ownership check. (#718) * Fix DnsRecord ownership check. * Var names --- .../deploy/webapp/deploy_webapp_from_registry.py | 4 ++-- .../deploy/webapp/undeploy_webapp_from_registry.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 2e4544eb..baaa31db 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -59,8 +59,8 @@ def process_app_deployment_request( dns_record = laconic.get_record(dns_crn) if dns_record: matched_owner = match_owner(app_deployment_request, dns_record) - if not matched_owner and dns_record.request: - matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.request, require=True)) + if not matched_owner and dns_record.attributes.request: + matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.attributes.request, require=True)) if matched_owner: print("Matched DnsRecord ownership:", matched_owner) diff --git a/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py index 6c5cac85..74cf0d60 100644 --- a/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py @@ -40,8 +40,8 @@ def process_app_removal_request(ctx, matched_owner = match_owner(app_removal_request, deployment_record, dns_record) # Or of the original deployment request. - if not matched_owner and deployment_record.request: - matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.request, require=True)) + if not matched_owner and deployment_record.attributes.request: + matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.attributes.request, require=True)) if matched_owner: print("Matched deployment ownership:", matched_owner) From 62af03077f44c0f9b853fe91f77228e9b1990d41 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 30 Jan 2024 22:13:45 -0600 Subject: [PATCH 08/27] Add deployed/error status output to the state file. (#719) * More status info * Up default resource limits. * Need ps --- .../container-build/cerc-nextjs-base/Dockerfile | 2 +- stack_orchestrator/deploy/k8s/cluster_info.py | 2 +- .../deploy/webapp/deploy_webapp_from_registry.py | 16 ++++++++++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile index 8949c4e9..c3c12cc6 100644 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile @@ -30,7 +30,7 @@ RUN \ # [Optional] Uncomment this section to install additional OS packages. RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends jq gettext-base + && apt-get -y install --no-install-recommends jq gettext-base procps # [Optional] Uncomment if you want to install more global node modules # RUN su node -c "npm install -g " diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 85fd63a8..9ba8041f 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -195,7 +195,7 @@ class ClusterInfo: volume_mounts=volume_mounts, resources=client.V1ResourceRequirements( requests={"cpu": "100m", "memory": "200Mi"}, - limits={"cpu": "500m", "memory": "500Mi"}, + limits={"cpu": "1000m", "memory": "2000Mi"}, ), ) containers.append(container) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index baaa31db..45f50377 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -136,13 +136,17 @@ def load_known_requests(filename): return {} -def dump_known_requests(filename, requests): +def dump_known_requests(filename, requests, status="SEEN"): if not filename: return known_requests = load_known_requests(filename) for r in requests: - known_requests[r.id] = r.createTime - json.dump(known_requests, open(filename, "w")) + known_requests[r.id] = { + "createTime": r.createTime, + "status": status + } + with open(filename, "w") as f: + json.dump(known_requests, f) @click.command() @@ -201,6 +205,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ requests.reverse() requests_by_name = {} for r in requests: + # TODO: Do this _after_ filtering deployments and cancellations to minimize round trips. app = laconic.get_record(r.attributes.application) if not app: print("Skipping request %s, cannot locate app." % r.id) @@ -256,6 +261,8 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ if not dry_run: for r in requests_to_execute: + dump_known_requests(state_file, [r], "DEPLOYING") + status = "ERROR" try: process_app_deployment_request( ctx, @@ -268,5 +275,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ kube_config, image_registry ) + status = "DEPLOYED" finally: - dump_known_requests(state_file, [r]) + dump_known_requests(state_file, [r], status) From 12ec1bec43163fc01ab59b66a8c896deb3806c49 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 30 Jan 2024 23:09:48 -0600 Subject: [PATCH 09/27] Add ConfigMap support for k8s. (#714) * Minor fixes for deploying with k8s and podman. * ConfigMap support --- stack_orchestrator/build/build_containers.py | 5 ++ .../deploy/deployment_create.py | 62 ++++++++++++-- stack_orchestrator/deploy/images.py | 3 +- stack_orchestrator/deploy/k8s/cluster_info.py | 80 ++++++++++++++----- stack_orchestrator/deploy/k8s/deploy_k8s.py | 75 ++++++++++++----- stack_orchestrator/deploy/k8s/helpers.py | 25 ++++-- stack_orchestrator/deploy/spec.py | 12 +++ 7 files changed, 208 insertions(+), 54 deletions(-) diff --git a/stack_orchestrator/build/build_containers.py b/stack_orchestrator/build/build_containers.py index e987c504..7b0957b0 100644 --- a/stack_orchestrator/build/build_containers.py +++ b/stack_orchestrator/build/build_containers.py @@ -33,6 +33,7 @@ from stack_orchestrator.base import get_npm_registry_url # TODO: find a place for this # epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)" + def make_container_build_env(dev_root_path: str, container_build_dir: str, debug: bool, @@ -104,6 +105,9 @@ def process_container(stack: str, build_command = os.path.join(container_build_dir, "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}" if not dry_run: + # No PATH at all causes failures with podman. + if "PATH" not in container_build_env: + container_build_env["PATH"] = os.environ["PATH"] if verbose: print(f"Executing: {build_command} with environment: {container_build_env}") build_result = subprocess.run(build_command, shell=True, env=container_build_env) @@ -119,6 +123,7 @@ def process_container(stack: str, else: print("Skipped") + @click.command() @click.option('--include', help="only build these containers") @click.option('--exclude', help="don\'t build these containers") diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index 138fc1bb..b95386b1 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -54,19 +54,44 @@ def _get_ports(stack): def _get_named_volumes(stack): # Parse the compose files looking for named volumes - named_volumes = [] + named_volumes = { + "rw": [], + "ro": [] + } parsed_stack = get_parsed_stack_config(stack) pods = get_pod_list(parsed_stack) yaml = get_yaml() + + def find_vol_usage(parsed_pod_file, vol): + ret = {} + if "services" in parsed_pod_file: + for svc_name, svc in parsed_pod_file["services"].items(): + if "volumes" in svc: + for svc_volume in svc["volumes"]: + parts = svc_volume.split(":") + if parts[0] == vol: + ret[svc_name] = { + "volume": parts[0], + "mount": parts[1], + "options": parts[2] if len(parts) == 3 else None + } + return ret + for pod in pods: pod_file_path = get_pod_file_path(parsed_stack, pod) parsed_pod_file = yaml.load(open(pod_file_path, "r")) if "volumes" in parsed_pod_file: volumes = parsed_pod_file["volumes"] for volume in volumes.keys(): - # Volume definition looks like: - # 'laconicd-data': None - named_volumes.append(volume) + for vu in find_vol_usage(parsed_pod_file, volume).values(): + read_only = vu["options"] == "ro" + if read_only: + if vu["volume"] not in named_volumes["rw"] and vu["volume"] not in named_volumes["ro"]: + named_volumes["ro"].append(vu["volume"]) + else: + if vu["volume"] not in named_volumes["rw"]: + named_volumes["rw"].append(vu["volume"]) + return named_volumes @@ -98,12 +123,24 @@ def _fixup_pod_file(pod, spec, compose_dir): _create_bind_dir_if_relative(volume, volume_spec, compose_dir) new_volume_spec = {"driver": "local", "driver_opts": { - "type": "none", - "device": volume_spec_fixedup, - "o": "bind" + "type": "none", + "device": volume_spec_fixedup, + "o": "bind" } } pod["volumes"][volume] = new_volume_spec + + # Fix up configmaps + if "configmaps" in spec: + spec_cfgmaps = spec["configmaps"] + if "volumes" in pod: + pod_volumes = pod["volumes"] + for volume in pod_volumes.keys(): + if volume in spec_cfgmaps: + volume_cfg = spec_cfgmaps[volume] + # Just make the dir (if necessary) + _create_bind_dir_if_relative(volume, volume_cfg, compose_dir) + # Fix up ports if "network" in spec and "ports" in spec["network"]: spec_ports = spec["network"]["ports"] @@ -319,9 +356,18 @@ def init_operation(deploy_command_context, stack, deployer_type, config, named_volumes = _get_named_volumes(stack) if named_volumes: volume_descriptors = {} - for named_volume in named_volumes: + configmap_descriptors = {} + for named_volume in named_volumes["rw"]: volume_descriptors[named_volume] = f"./data/{named_volume}" + for named_volume in named_volumes["ro"]: + if "k8s" in deployer_type: + if "config" in named_volume: + configmap_descriptors[named_volume] = f"./data/{named_volume}" + else: + volume_descriptors[named_volume] = f"./data/{named_volume}" spec_file_content["volumes"] = volume_descriptors + if configmap_descriptors: + spec_file_content["configmaps"] = configmap_descriptors if opts.o.debug: print(f"Creating spec file for stack: {stack} with content: {spec_file_content}") diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py index ddbb33f7..28d3bdb8 100644 --- a/stack_orchestrator/deploy/images.py +++ b/stack_orchestrator/deploy/images.py @@ -31,7 +31,8 @@ def _image_needs_pushed(image: str): def remote_tag_for_image(image: str, remote_repo_url: str): # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy - (org, image_name_with_version) = image.split("/") + major_parts = image.split("/", 2) + image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0] (image_name, image_version) = image_name_with_version.split(":") if image_version == "local": return f"{remote_repo_url}/{image_name}:deploy" diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 9ba8041f..f0e7c87b 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -13,6 +13,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import os + from kubernetes import client from typing import Any, List, Set @@ -112,9 +114,10 @@ class ClusterInfo: services = pod["services"] for service_name in services: service_info = services[service_name] - port = int(service_info["ports"][0]) - if opts.o.debug: - print(f"service port: {port}") + if "ports" in service_info: + port = int(service_info["ports"][0]) + if opts.o.debug: + print(f"service port: {port}") service = client.V1Service( metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"), spec=client.V1ServiceSpec( @@ -130,30 +133,70 @@ class ClusterInfo: def get_pvcs(self): result = [] - volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) + spec_volumes = self.spec.get_volumes() + named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) if opts.o.debug: - print(f"Volumes: {volumes}") - for volume_name in volumes: + print(f"Spec Volumes: {spec_volumes}") + print(f"Named Volumes: {named_volumes}") + for volume_name in spec_volumes: + if volume_name not in named_volumes: + if opts.o.debug: + print(f"{volume_name} not in pod files") + continue spec = client.V1PersistentVolumeClaimSpec( access_modes=["ReadWriteOnce"], storage_class_name="manual", resources=client.V1ResourceRequirements( requests={"storage": "2Gi"} ), - volume_name=volume_name + volume_name=f"{self.app_name}-{volume_name}" ) pvc = client.V1PersistentVolumeClaim( - metadata=client.V1ObjectMeta(name=volume_name, - labels={"volume-label": volume_name}), + metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}", + labels={"volume-label": f"{self.app_name}-{volume_name}"}), spec=spec, ) result.append(pvc) return result + def get_configmaps(self): + result = [] + spec_configmaps = self.spec.get_configmaps() + named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) + for cfg_map_name, cfg_map_path in spec_configmaps.items(): + if cfg_map_name not in named_volumes: + if opts.o.debug: + print(f"{cfg_map_name} not in pod files") + continue + + if not cfg_map_path.startswith("/"): + cfg_map_path = os.path.join(os.path.dirname(self.spec.file_path), cfg_map_path) + + # Read in all the files at a single-level of the directory. This mimics the behavior + # of `kubectl create configmap foo --from-file=/path/to/dir` + data = {} + for f in os.listdir(cfg_map_path): + full_path = os.path.join(cfg_map_path, f) + if os.path.isfile(full_path): + data[f] = open(full_path, 'rt').read() + + spec = client.V1ConfigMap( + metadata=client.V1ObjectMeta(name=f"{self.app_name}-{cfg_map_name}", + labels={"configmap-label": cfg_map_name}), + data=data + ) + result.append(spec) + return result + def get_pvs(self): result = [] - volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) - for volume_name in volumes: + spec_volumes = self.spec.get_volumes() + named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) + for volume_name in spec_volumes: + if volume_name not in named_volumes: + if opts.o.debug: + print(f"{volume_name} not in pod files") + continue spec = client.V1PersistentVolumeSpec( storage_class_name="manual", access_modes=["ReadWriteOnce"], @@ -161,8 +204,8 @@ class ClusterInfo: host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name)) ) pv = client.V1PersistentVolume( - metadata=client.V1ObjectMeta(name=volume_name, - labels={"volume-label": volume_name}), + metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}", + labels={"volume-label": f"{self.app_name}-{volume_name}"}), spec=spec, ) result.append(pv) @@ -178,10 +221,11 @@ class ClusterInfo: container_name = service_name service_info = services[service_name] image = service_info["image"] - port = int(service_info["ports"][0]) - if opts.o.debug: - print(f"image: {image}") - print(f"service port: {port}") + if "ports" in service_info: + port = int(service_info["ports"][0]) + if opts.o.debug: + print(f"image: {image}") + print(f"service port: {port}") # Re-write the image tag for remote deployment image_to_use = remote_tag_for_image( image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image @@ -199,7 +243,7 @@ class ClusterInfo: ), ) containers.append(container) - volumes = volumes_for_pod_files(self.parsed_pod_yaml_map) + volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec) image_pull_secrets = [client.V1LocalObjectReference(name="laconic-registry")] template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels={"app": self.app_name}), diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 0a339fe9..045d1893 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -1,5 +1,4 @@ # Copyright © 2023 Vulcanize - # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -110,6 +109,20 @@ class K8sDeployer(Deployer): if opts.o.debug: print("PVCs created:") print(f"{pvc_resp}") + + # Figure out the ConfigMaps for this deployment + config_maps = self.cluster_info.get_configmaps() + for cfg_map in config_maps: + if opts.o.debug: + print(f"Sending this ConfigMap: {cfg_map}") + cfg_rsp = self.core_api.create_namespaced_config_map( + body=cfg_map, + namespace=self.k8s_namespace + ) + if opts.o.debug: + print("ConfigMap created:") + print(f"{cfg_rsp}") + # Process compose files into a Deployment deployment = self.cluster_info.get_deployment(image_pull_policy=None if self.is_kind() else "Always") # Create the k8s objects @@ -135,17 +148,21 @@ class K8sDeployer(Deployer): if not self.is_kind(): ingress: client.V1Ingress = self.cluster_info.get_ingress() - if opts.o.debug: - print(f"Sending this ingress: {ingress}") - ingress_resp = self.networking_api.create_namespaced_ingress( - namespace=self.k8s_namespace, - body=ingress - ) - if opts.o.debug: - print("Ingress created:") - print(f"{ingress_resp}") + if ingress: + if opts.o.debug: + print(f"Sending this ingress: {ingress}") + ingress_resp = self.networking_api.create_namespaced_ingress( + namespace=self.k8s_namespace, + body=ingress + ) + if opts.o.debug: + print("Ingress created:") + print(f"{ingress_resp}") + else: + if opts.o.debug: + print("No ingress configured") - def down(self, timeout, volumes): + def down(self, timeout, volumes): # noqa: C901 self.connect_api() # Delete the k8s objects # Create the host-path-mounted PVs for this deployment @@ -175,6 +192,22 @@ class K8sDeployer(Deployer): print(f"{pvc_resp}") except client.exceptions.ApiException as e: _check_delete_exception(e) + + # Figure out the ConfigMaps for this deployment + cfg_maps = self.cluster_info.get_configmaps() + for cfg_map in cfg_maps: + if opts.o.debug: + print(f"Deleting this ConfigMap: {cfg_map}") + try: + cfg_map_resp = self.core_api.delete_namespaced_config_map( + name=cfg_map.metadata.name, namespace=self.k8s_namespace + ) + if opts.o.debug: + print("ConfigMap deleted:") + print(f"{cfg_map_resp}") + except client.exceptions.ApiException as e: + _check_delete_exception(e) + deployment = self.cluster_info.get_deployment() if opts.o.debug: print(f"Deleting this deployment: {deployment}") @@ -198,14 +231,18 @@ class K8sDeployer(Deployer): if not self.is_kind(): ingress: client.V1Ingress = self.cluster_info.get_ingress() - if opts.o.debug: - print(f"Deleting this ingress: {ingress}") - try: - self.networking_api.delete_namespaced_ingress( - name=ingress.metadata.name, namespace=self.k8s_namespace - ) - except client.exceptions.ApiException as e: - _check_delete_exception(e) + if ingress: + if opts.o.debug: + print(f"Deleting this ingress: {ingress}") + try: + self.networking_api.delete_namespaced_ingress( + name=ingress.metadata.name, namespace=self.k8s_namespace + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) + else: + if opts.o.debug: + print("No ingress to delete") if self.is_kind(): # Destroy the kind cluster diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index 62545dfd..b094acd5 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -73,7 +73,7 @@ def named_volumes_from_pod_files(parsed_pod_files): parsed_pod_file = parsed_pod_files[pod] if "volumes" in parsed_pod_file: volumes = parsed_pod_file["volumes"] - for volume in volumes.keys(): + for volume, value in volumes.items(): # Volume definition looks like: # 'laconicd-data': None named_volumes.append(volume) @@ -98,22 +98,31 @@ def volume_mounts_for_service(parsed_pod_files, service): volumes = service_obj["volumes"] for mount_string in volumes: # Looks like: test-data:/data - (volume_name, mount_path) = mount_string.split(":") - volume_device = client.V1VolumeMount(mount_path=mount_path, name=volume_name) + parts = mount_string.split(":") + volume_name = parts[0] + mount_path = parts[1] + mount_options = parts[2] if len(parts) == 3 else None + volume_device = client.V1VolumeMount( + mount_path=mount_path, name=volume_name, read_only="ro" == mount_options) result.append(volume_device) return result -def volumes_for_pod_files(parsed_pod_files): +def volumes_for_pod_files(parsed_pod_files, spec): result = [] for pod in parsed_pod_files: parsed_pod_file = parsed_pod_files[pod] if "volumes" in parsed_pod_file: volumes = parsed_pod_file["volumes"] for volume_name in volumes.keys(): - claim = client.V1PersistentVolumeClaimVolumeSource(claim_name=volume_name) - volume = client.V1Volume(name=volume_name, persistent_volume_claim=claim) - result.append(volume) + if volume_name in spec.get_configmaps(): + config_map = client.V1ConfigMapVolumeSource(name=volume_name) + volume = client.V1Volume(name=volume_name, config_map=config_map) + result.append(volume) + else: + claim = client.V1PersistentVolumeClaimVolumeSource(claim_name=volume_name) + volume = client.V1Volume(name=volume_name, persistent_volume_claim=claim) + result.append(volume) return result @@ -158,7 +167,7 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir): volume_definitions.append( f" - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n" f" containerPath: {get_node_pv_mount_path(volume_name)}" - ) + ) return ( "" if len(volume_definitions) == 0 else ( " extraMounts:\n" diff --git a/stack_orchestrator/deploy/spec.py b/stack_orchestrator/deploy/spec.py index c4f791bf..dd6cd107 100644 --- a/stack_orchestrator/deploy/spec.py +++ b/stack_orchestrator/deploy/spec.py @@ -22,6 +22,7 @@ from stack_orchestrator import constants class Spec: obj: typing.Any + file_path: Path def __init__(self) -> None: pass @@ -29,12 +30,23 @@ class Spec: def init_from_file(self, file_path: Path): with file_path: self.obj = get_yaml().load(open(file_path, "r")) + self.file_path = file_path def get_image_registry(self): return (self.obj[constants.image_resigtry_key] if self.obj and constants.image_resigtry_key in self.obj else None) + def get_volumes(self): + return (self.obj["volumes"] + if self.obj and "volumes" in self.obj + else {}) + + def get_configmaps(self): + return (self.obj["configmaps"] + if self.obj and "configmaps" in self.obj + else {}) + def get_http_proxy(self): return (self.obj[constants.network_key][constants.http_proxy_key] if self.obj and constants.network_key in self.obj From b39805078798abfae636ec4b7f277bf62a1bb5ed Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 31 Jan 2024 15:11:32 -0600 Subject: [PATCH 10/27] Don't include volumes in spec if we don't have any. (#720) --- stack_orchestrator/deploy/deployment_create.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index b95386b1..b72f262c 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -365,7 +365,8 @@ def init_operation(deploy_command_context, stack, deployer_type, config, configmap_descriptors[named_volume] = f"./data/{named_volume}" else: volume_descriptors[named_volume] = f"./data/{named_volume}" - spec_file_content["volumes"] = volume_descriptors + if volume_descriptors: + spec_file_content["volumes"] = volume_descriptors if configmap_descriptors: spec_file_content["configmaps"] = configmap_descriptors From 1c30441000a461a00dc2b3e1e160a054b0d60c07 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 1 Feb 2024 07:40:20 -0700 Subject: [PATCH 11/27] Add schedule for k8s deploy test --- .gitea/workflows/test-k8s-deploy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitea/workflows/test-k8s-deploy.yml b/.gitea/workflows/test-k8s-deploy.yml index 92c76a7d..50315363 100644 --- a/.gitea/workflows/test-k8s-deploy.yml +++ b/.gitea/workflows/test-k8s-deploy.yml @@ -8,6 +8,8 @@ on: - '.gitea/workflows/triggers/test-k8s-deploy' - '.gitea/workflows/test-k8s-deploy.yml' - 'tests/k8s-deploy/run-deploy-test.sh' + schedule: # Note: coordinate with other tests to not overload runners at the same time of day + - cron: '3 15 * * *' jobs: test: From 6629017d6a987f77fede8580843947b6715976dc Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 2 Feb 2024 18:04:06 -0600 Subject: [PATCH 12/27] Support other webapp types (react, static). (#721) * Support other webapp types (react, static). --- stack_orchestrator/build/build_webapp.py | 11 +++--- .../cerc-nextjs-base/Dockerfile | 2 +- .../scripts/start-serving-app.sh | 2 +- .../cerc-webapp-base/Dockerfile | 8 ++--- .../cerc-webapp-base/Dockerfile.webapp | 11 ++++++ .../container-build/cerc-webapp-base/build.sh | 24 +++++++++++-- .../scripts/apply-runtime-env.sh | 33 +++++++++++++++++ .../{ => scripts}/apply-webapp-config.sh | 0 .../cerc-webapp-base/scripts/build-app.sh | 36 +++++++++++++++++++ .../scripts/start-serving-app.sh | 15 ++++++++ .../cerc-webapp-base/start-serving-app.sh | 9 ----- .../deploy/webapp/run_webapp.py | 2 +- stack_orchestrator/deploy/webapp/util.py | 26 +++++++++++++- tests/webapp-test/run-webapp-test.sh | 4 +-- 14 files changed, 157 insertions(+), 26 deletions(-) create mode 100644 stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp create mode 100755 stack_orchestrator/data/container-build/cerc-webapp-base/scripts/apply-runtime-env.sh rename stack_orchestrator/data/container-build/cerc-webapp-base/{ => scripts}/apply-webapp-config.sh (100%) create mode 100755 stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh create mode 100755 stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh delete mode 100755 stack_orchestrator/data/container-build/cerc-webapp-base/start-serving-app.sh diff --git a/stack_orchestrator/build/build_webapp.py b/stack_orchestrator/build/build_webapp.py index 287347eb..a9124590 100644 --- a/stack_orchestrator/build/build_webapp.py +++ b/stack_orchestrator/build/build_webapp.py @@ -25,10 +25,11 @@ from decouple import config import click from pathlib import Path from stack_orchestrator.build import build_containers +from stack_orchestrator.deploy.webapp.util import determine_base_container @click.command() -@click.option('--base-container', default="cerc/nextjs-base") +@click.option('--base-container') @click.option('--source-repo', help="directory containing the webapp to build", required=True) @click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild") @click.option("--extra-build-args", help="Supply extra arguments to build") @@ -57,6 +58,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t if not quiet: print(f'Dev Root is: {dev_root_path}') + if not base_container: + base_container = determine_base_container(source_repo) + # First build the base container. container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug, force_rebuild, extra_build_args) @@ -64,13 +68,12 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet, verbose, dry_run, continue_on_error) - # Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir. container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true" container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo) container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir, - base_container.replace("/", "-"), - "Dockerfile.webapp") + base_container.replace("/", "-"), + "Dockerfile.webapp") if not tag: webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1] container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local" diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile index c3c12cc6..5f9548ee 100644 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # RUN su node -c "npm install -g " # Expose port for http -EXPOSE 3000 +EXPOSE 80 COPY /scripts /scripts diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh index 3692f2f6..bd254572 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh @@ -58,4 +58,4 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then fi fi -$CERC_BUILD_TOOL start . -p ${CERC_LISTEN_PORT:-3000} +$CERC_BUILD_TOOL start . -- -p ${CERC_LISTEN_PORT:-80} diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile index 275a5c3c..331e04a1 100644 --- a/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile @@ -1,6 +1,6 @@ # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster -ARG VARIANT=18-bullseye +ARG VARIANT=20-bullseye FROM node:${VARIANT} ARG USERNAME=node @@ -28,7 +28,7 @@ RUN \ # [Optional] Uncomment this section to install additional OS packages. RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends jq + && apt-get -y install --no-install-recommends jq gettext-base # [Optional] Uncomment if you want to install an additional version of node using nvm # ARG EXTRA_NODE_VERSION=10 @@ -37,9 +37,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # We do this to get a yq binary from the published container, for the correct architecture we're building here COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq -RUN mkdir -p /scripts -COPY ./apply-webapp-config.sh /scripts -COPY ./start-serving-app.sh /scripts +COPY scripts /scripts # [Optional] Uncomment if you want to install more global node modules # RUN su node -c "npm install -g " diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp new file mode 100644 index 00000000..711eff25 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp @@ -0,0 +1,11 @@ +FROM cerc/webapp-base:local as builder + +ARG CERC_BUILD_TOOL + +WORKDIR /app +COPY . . +RUN rm -rf node_modules build .next* +RUN /scripts/build-app.sh /app build /data + +FROM cerc/webapp-base:local +COPY --from=builder /data /data diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/build.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/build.sh index 51712dc4..12b25e5e 100755 --- a/stack_orchestrator/data/container-build/cerc-webapp-base/build.sh +++ b/stack_orchestrator/data/container-build/cerc-webapp-base/build.sh @@ -1,9 +1,29 @@ #!/usr/bin/env bash -# Build cerc/laconic-registry-cli +# Build cerc/webapp-base source ${CERC_CONTAINER_BASE_DIR}/build-base.sh # See: https://stackoverflow.com/a/246128/1701505 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -docker build -t cerc/webapp-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile ${SCRIPT_DIR} +CERC_CONTAINER_BUILD_WORK_DIR=${CERC_CONTAINER_BUILD_WORK_DIR:-$SCRIPT_DIR} +CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/Dockerfile} +CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/webapp-base:local} + +docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR + +if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/webapp-base:local" ]; then + cat < $TMP_ENV + set -a + source .env + source $TMP_ENV + set +a + rm -f $TMP_ENV +fi + +for f in $(find . -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do + for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},();]' '\n' | egrep -o -e '^"CERC_RUNTIME_ENV_[^\"]+"' -e '^"LACONIC_HOSTED_CONFIG_[^\"]+"'); do + orig_name=$(echo -n "${e}" | sed 's/"//g') + cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g') + cur_val=$(echo -n "\$${cur_name}" | envsubst) + if [ "$CERC_RETAIN_ENV_QUOTES" != "true" ]; then + cur_val=$(sed "s/^[\"']//" <<< "$cur_val" | sed "s/[\"']//") + fi + esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val") + echo "$f: $cur_name=$cur_val" + sed -i "s/$orig_name/$esc_val/g" $f + done +done diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/apply-webapp-config.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/apply-webapp-config.sh similarity index 100% rename from stack_orchestrator/data/container-build/cerc-webapp-base/apply-webapp-config.sh rename to stack_orchestrator/data/container-build/cerc-webapp-base/scripts/apply-webapp-config.sh diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh new file mode 100755 index 00000000..c102a054 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +CERC_BUILD_TOOL="${CERC_BUILD_TOOL}" +WORK_DIR="${1:-/app}" +OUTPUT_DIR="${2:-build}" +DEST_DIR="${3:-/data}" + +if [ -f "${WORK_DIR}/package.json" ]; then + echo "Building node-based webapp ..." + cd "${WORK_DIR}" || exit 1 + + if [ -z "$CERC_BUILD_TOOL" ]; then + if [ -f "yarn.lock" ]; then + CERC_BUILD_TOOL=yarn + else + CERC_BUILD_TOOL=npm + fi + fi + + $CERC_BUILD_TOOL install || exit 1 + $CERC_BUILD_TOOL build || exit 1 + + rm -rf "${DEST_DIR}" + mv "${WORK_DIR}/${OUTPUT_DIR}" "${DEST_DIR}" +else + echo "Copying static app ..." + mv "${WORK_DIR}" "${DEST_DIR}" +fi + +exit 0 diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh new file mode 100755 index 00000000..365d05fb --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}" +CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}" + +if [ "true" == "$CERC_ENABLE_CORS" ]; then + CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --cors" +fi + +/scripts/apply-webapp-config.sh /config/config.yml ${CERC_WEBAPP_FILES_DIR} +/scripts/apply-runtime-env.sh ${CERC_WEBAPP_FILES_DIR} +http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT:-80} ${CERC_WEBAPP_FILES_DIR} diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/start-serving-app.sh deleted file mode 100755 index 69fa6c22..00000000 --- a/stack_orchestrator/data/container-build/cerc-webapp-base/start-serving-app.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -if [ -n "$CERC_SCRIPT_DEBUG" ]; then - set -x -fi - -CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}" - -/scripts/apply-webapp-config.sh /config/config.yml ${CERC_WEBAPP_FILES_DIR} -http-server -p 80 ${CERC_WEBAPP_FILES_DIR} diff --git a/stack_orchestrator/deploy/webapp/run_webapp.py b/stack_orchestrator/deploy/webapp/run_webapp.py index 4dbf234a..f780c6f8 100644 --- a/stack_orchestrator/deploy/webapp/run_webapp.py +++ b/stack_orchestrator/deploy/webapp/run_webapp.py @@ -27,7 +27,7 @@ from dotenv import dotenv_values from stack_orchestrator import constants from stack_orchestrator.deploy.deployer_factory import getDeployer -WEBAPP_PORT = 3000 +WEBAPP_PORT = 80 @click.command() diff --git a/stack_orchestrator/deploy/webapp/util.py b/stack_orchestrator/deploy/webapp/util.py index ef98117b..50485110 100644 --- a/stack_orchestrator/deploy/webapp/util.py +++ b/stack_orchestrator/deploy/webapp/util.py @@ -195,6 +195,23 @@ def file_hash(filename): return hashlib.sha1(open(filename).read().encode()).hexdigest() +def determine_base_container(clone_dir, app_type="webapp"): + if not app_type or not app_type.startswith("webapp"): + raise Exception(f"Unsupported app_type {app_type}") + + base_container = "cerc/webapp-base" + if app_type == "webapp/next": + base_container = "cerc/nextjs-base" + elif app_type == "webapp": + pkg_json_path = os.path.join(clone_dir, "package.json") + if os.path.exists(pkg_json_path): + pkg_json = json.load(open(pkg_json_path)) + if "next" in pkg_json.get("dependencies", {}): + base_container = "cerc/nextjs-base" + + return base_container + + def build_container_image(app_record, tag, extra_build_args=[]): tmpdir = tempfile.mkdtemp() @@ -216,8 +233,15 @@ def build_container_image(app_record, tag, extra_build_args=[]): result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir]) result.check_returncode() + base_container = determine_base_container(clone_dir, app_record.attributes.app_type) + print("Building webapp ...") - build_command = [sys.argv[0], "build-webapp", "--source-repo", clone_dir, "--tag", tag] + build_command = [ + sys.argv[0], "build-webapp", + "--source-repo", clone_dir, + "--tag", tag, + "--base-container", base_container + ] if extra_build_args: build_command.append("--extra-build-args") build_command.append(" ".join(extra_build_args)) diff --git a/tests/webapp-test/run-webapp-test.sh b/tests/webapp-test/run-webapp-test.sh index 5db382f8..8cae4828 100755 --- a/tests/webapp-test/run-webapp-test.sh +++ b/tests/webapp-test/run-webapp-test.sh @@ -30,14 +30,14 @@ CHECK="SPECIAL_01234567890_TEST_STRING" set +e -CONTAINER_ID=$(docker run -p 3000:3000 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local) +CONTAINER_ID=$(docker run -p 3000:80 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local) sleep 3 wget -t 7 -O test.before -m http://localhost:3000 docker logs $CONTAINER_ID docker remove -f $CONTAINER_ID -CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local) +CONTAINER_ID=$(docker run -p 3000:80 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local) sleep 3 wget -t 7 -O test.after -m http://localhost:3000 From 2fcd416e29517e682f00e5d663b939b3ee2e1d86 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 2 Feb 2024 20:05:15 -0600 Subject: [PATCH 13/27] Basic webapp deployer stack. (#722) --- .../cerc-webapp-deployer-backend/build.sh | 9 +++++++++ .../data/stacks/webapp-deployer-backend/stack.yml | 11 +++++++++++ stack_orchestrator/deploy/deploy.py | 8 ++++---- stack_orchestrator/deploy/deployment_create.py | 9 ++++----- stack_orchestrator/deploy/k8s/cluster_info.py | 2 +- stack_orchestrator/deploy/k8s/helpers.py | 6 +++--- 6 files changed, 32 insertions(+), 13 deletions(-) create mode 100755 stack_orchestrator/data/container-build/cerc-webapp-deployer-backend/build.sh create mode 100644 stack_orchestrator/data/stacks/webapp-deployer-backend/stack.yml diff --git a/stack_orchestrator/data/container-build/cerc-webapp-deployer-backend/build.sh b/stack_orchestrator/data/container-build/cerc-webapp-deployer-backend/build.sh new file mode 100755 index 00000000..948701d6 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-webapp-deployer-backend/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Build cerc/webapp-deployer-backend + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/webapp-deployer-backend:local ${build_command_args} ${CERC_REPO_BASE_DIR}/webapp-deployment-status-api diff --git a/stack_orchestrator/data/stacks/webapp-deployer-backend/stack.yml b/stack_orchestrator/data/stacks/webapp-deployer-backend/stack.yml new file mode 100644 index 00000000..ca61049c --- /dev/null +++ b/stack_orchestrator/data/stacks/webapp-deployer-backend/stack.yml @@ -0,0 +1,11 @@ +version: "1.0" +name: webapp-deployer-backend +description: "Deployer for webapps" +repos: + - git.vdb.to:telackey/webapp-deployment-status-api +containers: + - cerc/webapp-deployer-backend +pods: + - name: webapp-deployer-backend + repository: git.vdb.to:telackey/webapp-deployment-status-api + path: ./ diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index 18d27a21..de68154b 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -347,8 +347,8 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): else: if deployment: compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml") - pod_pre_start_command = pod["pre_start_command"] - pod_post_start_command = pod["post_start_command"] + pod_pre_start_command = pod.get("pre_start_command") + pod_post_start_command = pod.get("post_start_command") script_dir = compose_dir.parent.joinpath("pods", pod_name, "scripts") if pod_pre_start_command is not None: pre_start_commands.append(os.path.join(script_dir, pod_pre_start_command)) @@ -357,8 +357,8 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): else: pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"]) compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml") - pod_pre_start_command = pod["pre_start_command"] - pod_post_start_command = pod["post_start_command"] + pod_pre_start_command = pod.get("pre_start_command") + pod_post_start_command = pod.get("post_start_command") if pod_pre_start_command is not None: pre_start_commands.append(os.path.join(pod_root_dir, pod_pre_start_command)) if pod_post_start_command is not None: diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index b72f262c..44824766 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -360,11 +360,10 @@ def init_operation(deploy_command_context, stack, deployer_type, config, for named_volume in named_volumes["rw"]: volume_descriptors[named_volume] = f"./data/{named_volume}" for named_volume in named_volumes["ro"]: - if "k8s" in deployer_type: - if "config" in named_volume: - configmap_descriptors[named_volume] = f"./data/{named_volume}" - else: - volume_descriptors[named_volume] = f"./data/{named_volume}" + if "k8s" in deployer_type and "config" in named_volume: + configmap_descriptors[named_volume] = f"./data/{named_volume}" + else: + volume_descriptors[named_volume] = f"./data/{named_volume}" if volume_descriptors: spec_file_content["volumes"] = volume_descriptors if configmap_descriptors: diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index f0e7c87b..35b2b9da 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -243,7 +243,7 @@ class ClusterInfo: ), ) containers.append(container) - volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec) + volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec, self.app_name) image_pull_secrets = [client.V1LocalObjectReference(name="laconic-registry")] template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels={"app": self.app_name}), diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index b094acd5..081c3e62 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -108,7 +108,7 @@ def volume_mounts_for_service(parsed_pod_files, service): return result -def volumes_for_pod_files(parsed_pod_files, spec): +def volumes_for_pod_files(parsed_pod_files, spec, app_name): result = [] for pod in parsed_pod_files: parsed_pod_file = parsed_pod_files[pod] @@ -116,11 +116,11 @@ def volumes_for_pod_files(parsed_pod_files, spec): volumes = parsed_pod_file["volumes"] for volume_name in volumes.keys(): if volume_name in spec.get_configmaps(): - config_map = client.V1ConfigMapVolumeSource(name=volume_name) + config_map = client.V1ConfigMapVolumeSource(name=f"{app_name}-{volume_name}") volume = client.V1Volume(name=volume_name, config_map=config_map) result.append(volume) else: - claim = client.V1PersistentVolumeClaimVolumeSource(claim_name=volume_name) + claim = client.V1PersistentVolumeClaimVolumeSource(claim_name=f"{app_name}-{volume_name}") volume = client.V1Volume(name=volume_name, persistent_volume_claim=claim) result.append(volume) return result From 25a2b70f2c8c0f4f5ef5c1044f7941d6fb8eeb5f Mon Sep 17 00:00:00 2001 From: David Boreham Date: Sat, 3 Feb 2024 18:25:02 -0700 Subject: [PATCH 14/27] Fix command in mainnet-eth docs --- stack_orchestrator/data/stacks/mainnet-eth/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack_orchestrator/data/stacks/mainnet-eth/README.md b/stack_orchestrator/data/stacks/mainnet-eth/README.md index 8f0dc1c4..2656b75b 100644 --- a/stack_orchestrator/data/stacks/mainnet-eth/README.md +++ b/stack_orchestrator/data/stacks/mainnet-eth/README.md @@ -18,7 +18,7 @@ $ laconic-so --stack mainnet-eth build-containers ``` $ laconic-so --stack mainnet-eth deploy init --map-ports-to-host any-same --output mainnet-eth-spec.yml -$ laconic-so deploy create --spec-file mainnet-eth-spec.yml --deployment-dir mainnet-eth-deployment +$ laconic-so deploy --stack mainnet-eth create --spec-file mainnet-eth-spec.yml --deployment-dir mainnet-eth-deployment ``` ## Start the stack ``` From 36bb06898392a02cfd16dffefca5070246002814 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Mon, 5 Feb 2024 14:15:11 -0600 Subject: [PATCH 15/27] Add ConfigMap test. (#726) * Add ConfigMap test. * eof * Minor tweak * Trigger test --------- Co-authored-by: David Boreham --- .gitea/workflows/triggers/test-k8s-deploy | 1 + .../data/compose/docker-compose-test.yml | 2 ++ .../cerc-test-container/run.sh | 15 +++++++++++++ stack_orchestrator/deploy/deployer_factory.py | 4 ++-- .../deploy/deployment_create.py | 2 +- stack_orchestrator/deploy/k8s/deploy_k8s.py | 5 +++-- stack_orchestrator/deploy/k8s/helpers.py | 22 ++++++++++--------- tests/k8s-deploy/run-deploy-test.sh | 18 +++++++++++++-- 8 files changed, 52 insertions(+), 17 deletions(-) diff --git a/.gitea/workflows/triggers/test-k8s-deploy b/.gitea/workflows/triggers/test-k8s-deploy index 519ff4ac..f3cf0624 100644 --- a/.gitea/workflows/triggers/test-k8s-deploy +++ b/.gitea/workflows/triggers/test-k8s-deploy @@ -1 +1,2 @@ Change this file to trigger running the test-k8s-deploy CI job +Trigger test on PR branch diff --git a/stack_orchestrator/data/compose/docker-compose-test.yml b/stack_orchestrator/data/compose/docker-compose-test.yml index 5fbf46d0..50151f65 100644 --- a/stack_orchestrator/data/compose/docker-compose-test.yml +++ b/stack_orchestrator/data/compose/docker-compose-test.yml @@ -7,8 +7,10 @@ services: CERC_TEST_PARAM_1: ${CERC_TEST_PARAM_1:-FAILED} volumes: - test-data:/data + - test-config:/config:ro ports: - "80" volumes: test-data: + test-config: diff --git a/stack_orchestrator/data/container-build/cerc-test-container/run.sh b/stack_orchestrator/data/container-build/cerc-test-container/run.sh index da0af7d5..01fb874b 100755 --- a/stack_orchestrator/data/container-build/cerc-test-container/run.sh +++ b/stack_orchestrator/data/container-build/cerc-test-container/run.sh @@ -17,5 +17,20 @@ fi if [ -n "$CERC_TEST_PARAM_1" ]; then echo "Test-param-1: ${CERC_TEST_PARAM_1}" fi + +if [ -d "/config" ]; then + echo "/config: EXISTS" + for f in /config/*; do + if [[ -f "$f" ]] || [[ -L "$f" ]]; then + echo "$f:" + cat "$f" + echo "" + echo "" + fi + done +else + echo "/config: does NOT EXIST" +fi + # Run nginx which will block here forever /usr/sbin/nginx -g "daemon off;" diff --git a/stack_orchestrator/deploy/deployer_factory.py b/stack_orchestrator/deploy/deployer_factory.py index 959c1b7a..2d01729e 100644 --- a/stack_orchestrator/deploy/deployer_factory.py +++ b/stack_orchestrator/deploy/deployer_factory.py @@ -18,11 +18,11 @@ from stack_orchestrator.deploy.k8s.deploy_k8s import K8sDeployer, K8sDeployerCon from stack_orchestrator.deploy.compose.deploy_docker import DockerDeployer, DockerDeployerConfigGenerator -def getDeployerConfigGenerator(type: str): +def getDeployerConfigGenerator(type: str, deployment_context): if type == "compose" or type is None: return DockerDeployerConfigGenerator(type) elif type == constants.k8s_deploy_type or type == constants.k8s_kind_deploy_type: - return K8sDeployerConfigGenerator(type) + return K8sDeployerConfigGenerator(type, deployment_context) else: print(f"ERROR: deploy-to {type} is not valid") diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index 44824766..bb9eab40 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -487,7 +487,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw deployment_context = DeploymentContext() deployment_context.init(deployment_dir_path) # Call the deployer to generate any deployer-specific files (e.g. for kind) - deployer_config_generator = getDeployerConfigGenerator(deployment_type) + deployer_config_generator = getDeployerConfigGenerator(deployment_type, deployment_context) # TODO: make deployment_dir_path a Path above deployer_config_generator.generate(deployment_dir_path) call_stack_deploy_create(deployment_context, [network_dir, initial_peers, deployment_command_context]) diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 045d1893..3ff50277 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -390,8 +390,9 @@ class K8sDeployer(Deployer): class K8sDeployerConfigGenerator(DeployerConfigGenerator): type: str - def __init__(self, type: str) -> None: + def __init__(self, type: str, deployment_context) -> None: self.type = type + self.deployment_context = deployment_context super().__init__() def generate(self, deployment_dir: Path): @@ -399,7 +400,7 @@ class K8sDeployerConfigGenerator(DeployerConfigGenerator): if self.type == "k8s-kind": # Check the file isn't already there # Get the config file contents - content = generate_kind_config(deployment_dir) + content = generate_kind_config(deployment_dir, self.deployment_context) if opts.o.debug: print(f"kind config is: {content}") config_file = deployment_dir.joinpath(constants.kind_config_filename) diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index 081c3e62..b0c81969 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -134,8 +134,9 @@ def _get_host_paths_for_volumes(parsed_pod_files): volumes = parsed_pod_file["volumes"] for volume_name in volumes.keys(): volume_definition = volumes[volume_name] - host_path = volume_definition["driver_opts"]["device"] - result[volume_name] = host_path + if volume_definition and "driver_opts" in volume_definition: + host_path = volume_definition["driver_opts"]["device"] + result[volume_name] = host_path return result @@ -147,7 +148,7 @@ def _make_absolute_host_path(data_mount_path: Path, deployment_dir: Path) -> Pat return Path.cwd().joinpath(deployment_dir.joinpath("compose").joinpath(data_mount_path)).resolve() -def _generate_kind_mounts(parsed_pod_files, deployment_dir): +def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context): volume_definitions = [] volume_host_path_map = _get_host_paths_for_volumes(parsed_pod_files) # Note these paths are relative to the location of the pod files (at present) @@ -163,11 +164,12 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir): volumes = service_obj["volumes"] for mount_string in volumes: # Looks like: test-data:/data - (volume_name, mount_path) = mount_string.split(":") - volume_definitions.append( - f" - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n" - f" containerPath: {get_node_pv_mount_path(volume_name)}" - ) + volume_name = mount_string.split(":")[0] + if volume_name not in deployment_context.spec.get_configmaps(): + volume_definitions.append( + f" - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n" + f" containerPath: {get_node_pv_mount_path(volume_name)}" + ) return ( "" if len(volume_definitions) == 0 else ( " extraMounts:\n" @@ -223,13 +225,13 @@ def envs_from_environment_variables_map(map: Mapping[str, str]) -> List[client.V # extraMounts: # - hostPath: /path/to/my/files # containerPath: /files -def generate_kind_config(deployment_dir: Path): +def generate_kind_config(deployment_dir: Path, deployment_context): compose_file_dir = deployment_dir.joinpath("compose") # TODO: this should come from the stack file, not this way pod_files = [p for p in compose_file_dir.iterdir() if p.is_file()] parsed_pod_files_map = parsed_pod_files_map_from_file_names(pod_files) port_mappings_yml = _generate_kind_port_mappings(parsed_pod_files_map) - mounts_yml = _generate_kind_mounts(parsed_pod_files_map, deployment_dir) + mounts_yml = _generate_kind_mounts(parsed_pod_files_map, deployment_dir, deployment_context) return ( "kind: Cluster\n" "apiVersion: kind.x-k8s.io/v1alpha4\n" diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh index de1a5f10..04008217 100755 --- a/tests/k8s-deploy/run-deploy-test.sh +++ b/tests/k8s-deploy/run-deploy-test.sh @@ -97,6 +97,10 @@ if [ ! "$create_file_content" == "create-command-output-data" ]; then echo "deploy create test: FAILED" exit 1 fi + +# Add a config file to be picked up by the ConfigMap before starting. +echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/data/test-config/test_config + echo "deploy create output file test: passed" # Try to start the deployment $TEST_TARGET_SO deployment --dir $test_deployment_dir start @@ -117,6 +121,16 @@ else echo "deployment config test: FAILED" delete_cluster_exit fi + +# Check that the ConfigMap is mounted and contains the expected content. +log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) +if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then + echo "deployment ConfigMap test: passed" +else + echo "deployment ConfigMap test: FAILED" + delete_cluster_exit +fi + # Stop then start again and check the volume was preserved $TEST_TARGET_SO deployment --dir $test_deployment_dir stop # Sleep a bit just in case @@ -125,8 +139,8 @@ sleep 20 $TEST_TARGET_SO deployment --dir $test_deployment_dir start wait_for_pods_started wait_for_log_output -log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) -if [[ "$log_output_4" == *"Filesystem is old"* ]]; then +log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) +if [[ "$log_output_5" == *"Filesystem is old"* ]]; then echo "Retain volumes test: passed" else echo "Retain volumes test: FAILED" From 6848fc33cf9bc5005ca981c1b2de724c21dbe8a0 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 6 Feb 2024 07:07:56 -0700 Subject: [PATCH 16/27] Implement dry run support for k8s deploy (#727) --- .../deploy/compose/deploy_docker.py | 86 ++++++++------- stack_orchestrator/deploy/deploy.py | 67 ++++------- stack_orchestrator/deploy/k8s/deploy_k8s.py | 104 ++++++++++-------- 3 files changed, 132 insertions(+), 125 deletions(-) diff --git a/stack_orchestrator/deploy/compose/deploy_docker.py b/stack_orchestrator/deploy/compose/deploy_docker.py index b2622820..ffde91c2 100644 --- a/stack_orchestrator/deploy/compose/deploy_docker.py +++ b/stack_orchestrator/deploy/compose/deploy_docker.py @@ -17,6 +17,7 @@ from pathlib import Path from python_on_whales import DockerClient, DockerException from stack_orchestrator.deploy.deployer import Deployer, DeployerException, DeployerConfigGenerator from stack_orchestrator.deploy.deployment_context import DeploymentContext +from stack_orchestrator.opts import opts class DockerDeployer(Deployer): @@ -29,60 +30,69 @@ class DockerDeployer(Deployer): self.type = type def up(self, detach, services): - try: - return self.docker.compose.up(detach=detach, services=services) - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.compose.up(detach=detach, services=services) + except DockerException as e: + raise DeployerException(e) def down(self, timeout, volumes): - try: - return self.docker.compose.down(timeout=timeout, volumes=volumes) - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.compose.down(timeout=timeout, volumes=volumes) + except DockerException as e: + raise DeployerException(e) def update(self): - try: - return self.docker.compose.restart() - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.compose.restart() + except DockerException as e: + raise DeployerException(e) def status(self): - try: - for p in self.docker.compose.ps(): - print(f"{p.name}\t{p.state.status}") - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + for p in self.docker.compose.ps(): + print(f"{p.name}\t{p.state.status}") + except DockerException as e: + raise DeployerException(e) def ps(self): - try: - return self.docker.compose.ps() - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.compose.ps() + except DockerException as e: + raise DeployerException(e) def port(self, service, private_port): - try: - return self.docker.compose.port(service=service, private_port=private_port) - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.compose.port(service=service, private_port=private_port) + except DockerException as e: + raise DeployerException(e) def execute(self, service, command, tty, envs): - try: - return self.docker.compose.execute(service=service, command=command, tty=tty, envs=envs) - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.compose.execute(service=service, command=command, tty=tty, envs=envs) + except DockerException as e: + raise DeployerException(e) def logs(self, services, tail, follow, stream): - try: - return self.docker.compose.logs(services=services, tail=tail, follow=follow, stream=stream) - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.compose.logs(services=services, tail=tail, follow=follow, stream=stream) + except DockerException as e: + raise DeployerException(e) def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False): - try: - return self.docker.run(image=image, command=command, user=user, volumes=volumes, - entrypoint=entrypoint, envs=env, detach=detach, publish=ports, publish_all=len(ports) == 0) - except DockerException as e: - raise DeployerException(e) + if not opts.o.dry_run: + try: + return self.docker.run(image=image, command=command, user=user, volumes=volumes, + entrypoint=entrypoint, envs=env, detach=detach, publish=ports, publish_all=len(ports) == 0) + except DockerException as e: + raise DeployerException(e) class DockerDeployerConfigGenerator(DeployerConfigGenerator): diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index de68154b..29afcf13 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -85,54 +85,39 @@ def create_deploy_context( def up_operation(ctx, services_list, stay_attached=False): global_context = ctx.parent.parent.obj deploy_context = ctx.obj - if not global_context.dry_run: - cluster_context = deploy_context.cluster_context - container_exec_env = _make_runtime_env(global_context) - for attr, value in container_exec_env.items(): - os.environ[attr] = value - if global_context.verbose: - print(f"Running compose up with container_exec_env: {container_exec_env}, extra_args: {services_list}") - for pre_start_command in cluster_context.pre_start_commands: - _run_command(global_context, cluster_context.cluster, pre_start_command) - deploy_context.deployer.up(detach=not stay_attached, services=services_list) - for post_start_command in cluster_context.post_start_commands: - _run_command(global_context, cluster_context.cluster, post_start_command) - _orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.deployer, container_exec_env) + cluster_context = deploy_context.cluster_context + container_exec_env = _make_runtime_env(global_context) + for attr, value in container_exec_env.items(): + os.environ[attr] = value + if global_context.verbose: + print(f"Running compose up with container_exec_env: {container_exec_env}, extra_args: {services_list}") + for pre_start_command in cluster_context.pre_start_commands: + _run_command(global_context, cluster_context.cluster, pre_start_command) + deploy_context.deployer.up(detach=not stay_attached, services=services_list) + for post_start_command in cluster_context.post_start_commands: + _run_command(global_context, cluster_context.cluster, post_start_command) + _orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.deployer, container_exec_env) def down_operation(ctx, delete_volumes, extra_args_list): - global_context = ctx.parent.parent.obj - if not global_context.dry_run: - if global_context.verbose: - print("Running compose down") - timeout_arg = None - if extra_args_list: - timeout_arg = extra_args_list[0] - # Specify shutdown timeout (default 10s) to give services enough time to shutdown gracefully - ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes) + timeout_arg = None + if extra_args_list: + timeout_arg = extra_args_list[0] + # Specify shutdown timeout (default 10s) to give services enough time to shutdown gracefully + ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes) def status_operation(ctx): - global_context = ctx.parent.parent.obj - if not global_context.dry_run: - if global_context.verbose: - print("Running compose status") - ctx.obj.deployer.status() + ctx.obj.deployer.status() def update_operation(ctx): - global_context = ctx.parent.parent.obj - if not global_context.dry_run: - if global_context.verbose: - print("Running compose update") - ctx.obj.deployer.update() + ctx.obj.deployer.update() def ps_operation(ctx): global_context = ctx.parent.parent.obj if not global_context.dry_run: - if global_context.verbose: - print("Running compose ps") container_list = ctx.obj.deployer.ps() if len(container_list) > 0: print("Running containers:") @@ -187,15 +172,11 @@ def exec_operation(ctx, extra_args): def logs_operation(ctx, tail: int, follow: bool, extra_args: str): - global_context = ctx.parent.parent.obj extra_args_list = list(extra_args) or None - if not global_context.dry_run: - if global_context.verbose: - print("Running compose logs") - services_list = extra_args_list if extra_args_list is not None else [] - logs_stream = ctx.obj.deployer.logs(services=services_list, tail=tail, follow=follow, stream=True) - for stream_type, stream_content in logs_stream: - print(stream_content.decode("utf-8"), end="") + services_list = extra_args_list if extra_args_list is not None else [] + logs_stream = ctx.obj.deployer.logs(services=services_list, tail=tail, follow=follow, stream=True) + for stream_type, stream_content in logs_stream: + print(stream_content.decode("utf-8"), end="") @command.command() @@ -463,7 +444,7 @@ def _orchestrate_cluster_config(ctx, cluster_config, deployer, container_exec_en tty=False, envs=container_exec_env) waiting_for_data = False - if ctx.debug: + if ctx.debug and not waiting_for_data: print(f"destination output: {destination_output}") diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 3ff50277..e1f729c5 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -81,69 +81,84 @@ class K8sDeployer(Deployer): self.apps_api = client.AppsV1Api() self.custom_obj_api = client.CustomObjectsApi() - def up(self, detach, services): - - if self.is_kind(): - # Create the kind cluster - create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath(constants.kind_config_filename)) - # Ensure the referenced containers are copied into kind - load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set) - self.connect_api() - + def _create_volume_data(self): # Create the host-path-mounted PVs for this deployment pvs = self.cluster_info.get_pvs() for pv in pvs: if opts.o.debug: print(f"Sending this pv: {pv}") - pv_resp = self.core_api.create_persistent_volume(body=pv) - if opts.o.debug: - print("PVs created:") - print(f"{pv_resp}") + if not opts.o.dry_run: + pv_resp = self.core_api.create_persistent_volume(body=pv) + if opts.o.debug: + print("PVs created:") + print(f"{pv_resp}") # Figure out the PVCs for this deployment pvcs = self.cluster_info.get_pvcs() for pvc in pvcs: if opts.o.debug: print(f"Sending this pvc: {pvc}") - pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace=self.k8s_namespace) - if opts.o.debug: - print("PVCs created:") - print(f"{pvc_resp}") + + if not opts.o.dry_run: + pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace=self.k8s_namespace) + if opts.o.debug: + print("PVCs created:") + print(f"{pvc_resp}") # Figure out the ConfigMaps for this deployment config_maps = self.cluster_info.get_configmaps() for cfg_map in config_maps: if opts.o.debug: print(f"Sending this ConfigMap: {cfg_map}") - cfg_rsp = self.core_api.create_namespaced_config_map( - body=cfg_map, - namespace=self.k8s_namespace - ) - if opts.o.debug: - print("ConfigMap created:") - print(f"{cfg_rsp}") + if not opts.o.dry_run: + cfg_rsp = self.core_api.create_namespaced_config_map( + body=cfg_map, + namespace=self.k8s_namespace + ) + if opts.o.debug: + print("ConfigMap created:") + print(f"{cfg_rsp}") + def _create_deployment(self): # Process compose files into a Deployment deployment = self.cluster_info.get_deployment(image_pull_policy=None if self.is_kind() else "Always") # Create the k8s objects if opts.o.debug: print(f"Sending this deployment: {deployment}") - deployment_resp = self.apps_api.create_namespaced_deployment( - body=deployment, namespace=self.k8s_namespace - ) - if opts.o.debug: - print("Deployment created:") - print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \ - {deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}") + if not opts.o.dry_run: + deployment_resp = self.apps_api.create_namespaced_deployment( + body=deployment, namespace=self.k8s_namespace + ) + if opts.o.debug: + print("Deployment created:") + print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \ + {deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}") service: client.V1Service = self.cluster_info.get_service() - service_resp = self.core_api.create_namespaced_service( - namespace=self.k8s_namespace, - body=service - ) if opts.o.debug: - print("Service created:") - print(f"{service_resp}") + print(f"Sending this service: {service}") + if not opts.o.dry_run: + service_resp = self.core_api.create_namespaced_service( + namespace=self.k8s_namespace, + body=service + ) + if opts.o.debug: + print("Service created:") + print(f"{service_resp}") + + def up(self, detach, services): + if not opts.o.dry_run: + if self.is_kind(): + # Create the kind cluster + create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath(constants.kind_config_filename)) + # Ensure the referenced containers are copied into kind + load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set) + self.connect_api() + else: + print("Dry run mode enabled, skipping k8s API connect") + + self._create_volume_data() + self._create_deployment() if not self.is_kind(): ingress: client.V1Ingress = self.cluster_info.get_ingress() @@ -151,13 +166,14 @@ class K8sDeployer(Deployer): if ingress: if opts.o.debug: print(f"Sending this ingress: {ingress}") - ingress_resp = self.networking_api.create_namespaced_ingress( - namespace=self.k8s_namespace, - body=ingress - ) - if opts.o.debug: - print("Ingress created:") - print(f"{ingress_resp}") + if not opts.o.dry_run: + ingress_resp = self.networking_api.create_namespaced_ingress( + namespace=self.k8s_namespace, + body=ingress + ) + if opts.o.debug: + print("Ingress created:") + print(f"{ingress_resp}") else: if opts.o.debug: print("No ingress configured") From 3d5ececba5133096738df613f7cadb40cd7efcaa Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 6 Feb 2024 11:59:37 -0700 Subject: [PATCH 17/27] Remove duplicate plugin paths and resulting extraneous error message (#728) --- stack_orchestrator/util.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/stack_orchestrator/util.py b/stack_orchestrator/util.py index 0bd1a609..ce176848 100644 --- a/stack_orchestrator/util.py +++ b/stack_orchestrator/util.py @@ -19,7 +19,7 @@ import sys import ruamel.yaml from pathlib import Path from dotenv import dotenv_values -from typing import Mapping +from typing import Mapping, Set, List def include_exclude_check(s, include, exclude): @@ -81,17 +81,17 @@ def get_pod_list(parsed_stack): return result -def get_plugin_code_paths(stack): +def get_plugin_code_paths(stack) -> List[Path]: parsed_stack = get_parsed_stack_config(stack) pods = parsed_stack["pods"] - result = [] + result: Set[Path] = set() for pod in pods: if type(pod) is str: - result.append(get_stack_file_path(stack).parent) + result.add(get_stack_file_path(stack).parent) else: pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"]) - result.append(Path(os.path.join(pod_root_dir, "stack"))) - return result + result.add(Path(os.path.join(pod_root_dir, "stack"))) + return list(result) def get_pod_file_path(parsed_stack, pod_name: str): From bfbcfb7904ae951794fdf658bac7b33d1c278a8e Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 6 Feb 2024 12:32:10 -0700 Subject: [PATCH 18/27] Volume processing fixes (#729) --- stack_orchestrator/deploy/k8s/helpers.py | 32 +++++++++++++++++------- stack_orchestrator/util.py | 7 ++++++ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index b0c81969..e386b353 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -97,11 +97,17 @@ def volume_mounts_for_service(parsed_pod_files, service): if "volumes" in service_obj: volumes = service_obj["volumes"] for mount_string in volumes: - # Looks like: test-data:/data - parts = mount_string.split(":") - volume_name = parts[0] - mount_path = parts[1] - mount_options = parts[2] if len(parts) == 3 else None + # Looks like: test-data:/data or test-data:/data:ro or test-data:/data:rw + if opts.o.debug: + print(f"mount_string: {mount_string}") + mount_split = mount_string.split(":") + volume_name = mount_split[0] + mount_path = mount_split[1] + mount_options = mount_split[2] if len(mount_split) == 3 else None + if opts.o.debug: + print(f"volumne_name: {volume_name}") + print(f"mount path: {mount_path}") + print(f"mount options: {mount_options}") volume_device = client.V1VolumeMount( mount_path=mount_path, name=volume_name, read_only="ro" == mount_options) result.append(volume_device) @@ -163,12 +169,20 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context): if "volumes" in service_obj: volumes = service_obj["volumes"] for mount_string in volumes: - # Looks like: test-data:/data - volume_name = mount_string.split(":")[0] + # Looks like: test-data:/data or test-data:/data:ro or test-data:/data:rw + if opts.o.debug: + print(f"mount_string: {mount_string}") + mount_split = mount_string.split(":") + volume_name = mount_split[0] + mount_path = mount_split[1] + if opts.o.debug: + print(f"volumne_name: {volume_name}") + print(f"map: {volume_host_path_map}") + print(f"mount path: {mount_path}") if volume_name not in deployment_context.spec.get_configmaps(): volume_definitions.append( f" - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n" - f" containerPath: {get_node_pv_mount_path(volume_name)}" + f" containerPath: {get_node_pv_mount_path(volume_name)}\n" ) return ( "" if len(volume_definitions) == 0 else ( @@ -191,7 +205,7 @@ def _generate_kind_port_mappings(parsed_pod_files): for port_string in ports: # TODO handle the complex cases # Looks like: 80 or something more complicated - port_definitions.append(f" - containerPort: {port_string}\n hostPort: {port_string}") + port_definitions.append(f" - containerPort: {port_string}\n hostPort: {port_string}\n") return ( "" if len(port_definitions) == 0 else ( " extraPortMappings:\n" diff --git a/stack_orchestrator/util.py b/stack_orchestrator/util.py index ce176848..36c6bfd0 100644 --- a/stack_orchestrator/util.py +++ b/stack_orchestrator/util.py @@ -139,6 +139,13 @@ def get_compose_file_dir(): return source_compose_dir +def get_config_file_dir(): + # TODO: refactor to use common code with deploy command + data_dir = Path(__file__).absolute().parent.joinpath("data") + source_config_dir = data_dir.joinpath("config") + return source_config_dir + + def get_parsed_deployment_spec(spec_file): spec_file_path = Path(spec_file) try: From 937b983ec99ee06c7198efbf278452881942fc05 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 6 Feb 2024 22:42:44 +0000 Subject: [PATCH 19/27] Update links from github.com to git.vdb.to (#732) Update links and references to github.com to git.vdb.to. Also enable the flake8 lint action in gitea. Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/732 Co-authored-by: Thomas E Lackey Co-committed-by: Thomas E Lackey --- .gitea/workflows/lint.yml | 21 +++++++++++++++++++ README.md | 6 +++--- docs/CONTRIBUTING.md | 2 +- docs/adding-a-new-stack.md | 4 ++-- docs/spec.md | 2 +- scripts/cloud-init-dev-mode-install.yaml | 2 +- scripts/cloud-init-user-mode-install.yaml | 2 +- scripts/quick-install-linux.sh | 2 +- setup.py | 2 +- .../data/stacks/fixturenet-eth/README.md | 2 +- .../fixturenet-laconic-loaded/README.md | 4 ++-- .../data/stacks/fixturenet-laconicd/README.md | 4 ++-- .../data/stacks/mobymask/README.md | 2 +- 13 files changed, 38 insertions(+), 17 deletions(-) create mode 100644 .gitea/workflows/lint.yml diff --git a/.gitea/workflows/lint.yml b/.gitea/workflows/lint.yml new file mode 100644 index 00000000..f9be1e6b --- /dev/null +++ b/.gitea/workflows/lint.yml @@ -0,0 +1,21 @@ +name: Lint Checks + +on: + pull_request: + branches: '*' + push: + branches: '*' + +jobs: + test: + name: "Run linter" + runs-on: ubuntu-latest + steps: + - name: "Clone project repository" + uses: actions/checkout@v3 + - name: "Install Python" + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name : "Run flake8" + uses: py-actions/flake8@v2 diff --git a/README.md b/README.md index aa979e3a..ef504295 100644 --- a/README.md +++ b/README.md @@ -29,10 +29,10 @@ chmod +x ~/.docker/cli-plugins/docker-compose Next decide on a directory where you would like to put the stack-orchestrator program. Typically this would be a "user" binary directory such as `~/bin` or perhaps `/usr/local/laconic` or possibly just the current working directory. -Now, having selected that directory, download the latest release from [this page](https://github.com/cerc-io/stack-orchestrator/tags) into it (we're using `~/bin` below for concreteness but edit to suit if you selected a different directory). Also be sure that the destination directory exists and is writable: +Now, having selected that directory, download the latest release from [this page](https://git.vdb.to/cerc-io/stack-orchestrator/tags) into it (we're using `~/bin` below for concreteness but edit to suit if you selected a different directory). Also be sure that the destination directory exists and is writable: ```bash -curl -L -o ~/bin/laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so +curl -L -o ~/bin/laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so ``` Give it execute permissions: @@ -52,7 +52,7 @@ Version: 1.1.0-7a607c2-202304260513 Save the distribution url to `~/.laconic-so/config.yml`: ```bash mkdir ~/.laconic-so -echo "distribution-url: https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so" > ~/.laconic-so/config.yml +echo "distribution-url: https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so" > ~/.laconic-so/config.yml ``` ### Update diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 39b294fb..375a2d05 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -26,7 +26,7 @@ In addition to the pre-requisites listed in the [README](/README.md), the follow 1. Clone this repository: ``` - $ git clone https://github.com/cerc-io/stack-orchestrator.git + $ git clone https://git.vdb.to/cerc-io/stack-orchestrator.git ``` 2. Enter the project directory: diff --git a/docs/adding-a-new-stack.md b/docs/adding-a-new-stack.md index 2b2d1a65..1e42420e 100644 --- a/docs/adding-a-new-stack.md +++ b/docs/adding-a-new-stack.md @@ -1,10 +1,10 @@ # Adding a new stack -See [this PR](https://github.com/cerc-io/stack-orchestrator/pull/434) for an example of how to currently add a minimal stack to stack orchestrator. The [reth stack](https://github.com/cerc-io/stack-orchestrator/pull/435) is another good example. +See [this PR](https://git.vdb.to/cerc-io/stack-orchestrator/pull/434) for an example of how to currently add a minimal stack to stack orchestrator. The [reth stack](https://git.vdb.to/cerc-io/stack-orchestrator/pull/435) is another good example. For external developers, we recommend forking this repo and adding your stack directly to your fork. This initially requires running in "developer mode" as described [here](/docs/CONTRIBUTING.md). Check out the [Namada stack](https://github.com/vknowable/stack-orchestrator/blob/main/app/data/stacks/public-namada/digitalocean_quickstart.md) from Knowable to see how that is done. -Core to the feature completeness of stack orchestrator is to [decouple the tool functionality from payload](https://github.com/cerc-io/stack-orchestrator/issues/315) which will no longer require forking to add a stack. +Core to the feature completeness of stack orchestrator is to [decouple the tool functionality from payload](https://git.vdb.to/cerc-io/stack-orchestrator/issues/315) which will no longer require forking to add a stack. ## Example diff --git a/docs/spec.md b/docs/spec.md index 1dc9ac62..aa09274c 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1,6 +1,6 @@ # Specification -Note: this page is out of date (but still useful) - it will no longer be useful once stacks are [decoupled from the tool functionality](https://github.com/cerc-io/stack-orchestrator/issues/315). +Note: this page is out of date (but still useful) - it will no longer be useful once stacks are [decoupled from the tool functionality](https://git.vdb.to/cerc-io/stack-orchestrator/issues/315). ## Implementation diff --git a/scripts/cloud-init-dev-mode-install.yaml b/scripts/cloud-init-dev-mode-install.yaml index 965afe3a..7f9d9bd8 100644 --- a/scripts/cloud-init-dev-mode-install.yaml +++ b/scripts/cloud-init-dev-mode-install.yaml @@ -41,4 +41,4 @@ runcmd: - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - systemctl enable docker - systemctl start docker - - git clone https://github.com/cerc-io/stack-orchestrator.git /home/ubuntu/stack-orchestrator + - git clone https://git.vdb.to/cerc-io/stack-orchestrator.git /home/ubuntu/stack-orchestrator diff --git a/scripts/cloud-init-user-mode-install.yaml b/scripts/cloud-init-user-mode-install.yaml index bd02416c..f98f21bc 100644 --- a/scripts/cloud-init-user-mode-install.yaml +++ b/scripts/cloud-init-user-mode-install.yaml @@ -31,5 +31,5 @@ runcmd: - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - systemctl enable docker - systemctl start docker - - curl -L -o /usr/local/bin/laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so + - curl -L -o /usr/local/bin/laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so - chmod +x /usr/local/bin/laconic-so diff --git a/scripts/quick-install-linux.sh b/scripts/quick-install-linux.sh index 5670a403..b8642416 100755 --- a/scripts/quick-install-linux.sh +++ b/scripts/quick-install-linux.sh @@ -137,7 +137,7 @@ fi echo "**************************************************************************************" echo "Installing laconic-so" # install latest `laconic-so` -distribution_url=https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so +distribution_url=https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so install_filename=${install_dir}/laconic-so mkdir -p ${install_dir} curl -L -o ${install_filename} ${distribution_url} diff --git a/setup.py b/setup.py index d89dfc4d..773451f5 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setup( description='Orchestrates deployment of the Laconic stack', long_description=long_description, long_description_content_type="text/markdown", - url='https://github.com/cerc-io/stack-orchestrator', + url='https://git.vdb.to/cerc-io/stack-orchestrator', py_modules=['stack_orchestrator'], packages=find_packages(), install_requires=[requirements], diff --git a/stack_orchestrator/data/stacks/fixturenet-eth/README.md b/stack_orchestrator/data/stacks/fixturenet-eth/README.md index 853ab90c..6f1ec8e8 100644 --- a/stack_orchestrator/data/stacks/fixturenet-eth/README.md +++ b/stack_orchestrator/data/stacks/fixturenet-eth/README.md @@ -1,6 +1,6 @@ # fixturenet-eth -Instructions for deploying a local a geth + lighthouse blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator)): +Instructions for deploying a local a geth + lighthouse blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator (the installation of which is covered [here](https://git.vdb.to/cerc-io/stack-orchestrator)): ## Clone required repositories diff --git a/stack_orchestrator/data/stacks/fixturenet-laconic-loaded/README.md b/stack_orchestrator/data/stacks/fixturenet-laconic-loaded/README.md index c82e48ad..94f9eb36 100644 --- a/stack_orchestrator/data/stacks/fixturenet-laconic-loaded/README.md +++ b/stack_orchestrator/data/stacks/fixturenet-laconic-loaded/README.md @@ -7,11 +7,11 @@ Instructions for deploying a local Laconic blockchain "fixturenet" for developme **Note:** For building some NPMs, access to the @lirewine repositories is required. If you don't have access, see [this tutorial](/docs/laconicd-fixturenet.md) to run this stack ## 1. Install Laconic Stack Orchestrator -Installation is covered in detail [here](https://github.com/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as: +Installation is covered in detail [here](https://git.vdb.to/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as: ``` $ mkdir my-working-dir $ cd my-working-dir -$ curl -L -o ./laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so +$ curl -L -o ./laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so $ chmod +x ./laconic-so $ export PATH=$PATH:$(pwd) # Or move laconic-so to ~/bin or your favorite on-path directory ``` diff --git a/stack_orchestrator/data/stacks/fixturenet-laconicd/README.md b/stack_orchestrator/data/stacks/fixturenet-laconicd/README.md index 77a017a2..1b04b875 100644 --- a/stack_orchestrator/data/stacks/fixturenet-laconicd/README.md +++ b/stack_orchestrator/data/stacks/fixturenet-laconicd/README.md @@ -3,11 +3,11 @@ Instructions for deploying a local Laconic blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator. ## 1. Install Laconic Stack Orchestrator -Installation is covered in detail [here](https://github.com/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as: +Installation is covered in detail [here](https://git.vdb.to/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as: ``` $ mkdir my-working-dir $ cd my-working-dir -$ curl -L -o ./laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so +$ curl -L -o ./laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so $ chmod +x ./laconic-so $ export PATH=$PATH:$(pwd) # Or move laconic-so to ~/bin or your favorite on-path directory ``` diff --git a/stack_orchestrator/data/stacks/mobymask/README.md b/stack_orchestrator/data/stacks/mobymask/README.md index ef222cce..92a21c9d 100644 --- a/stack_orchestrator/data/stacks/mobymask/README.md +++ b/stack_orchestrator/data/stacks/mobymask/README.md @@ -4,7 +4,7 @@ The MobyMask watcher is a Laconic Network component that provides efficient acce ## Deploy the MobyMask Watcher -The instructions below show how to deploy a MobyMask watcher using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator#install)). +The instructions below show how to deploy a MobyMask watcher using laconic-stack-orchestrator (the installation of which is covered [here](https://git.vdb.to/cerc-io/stack-orchestrator#install)). This deployment expects that ipld-eth-server's endpoints are available on the local machine at http://ipld-eth-server.example.com:8083/graphql and http://ipld-eth-server.example.com:8082. More advanced configurations are supported by modifying the watcher's [config file](../../config/watcher-mobymask/mobymask-watcher.toml). From 88a0236ca9ce802b15b9da30a2dfc2dfff5fc42c Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 7 Feb 2024 03:12:40 +0000 Subject: [PATCH 20/27] Add the ability to filter deployment requests by tag. (#730) Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/730 --- .../webapp/deploy_webapp_from_registry.py | 74 ++++++++++++++----- stack_orchestrator/deploy/webapp/util.py | 20 ++--- 2 files changed, 68 insertions(+), 26 deletions(-) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 45f50377..4540ceac 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -19,6 +19,8 @@ import shlex import shutil import sys import tempfile +import time +import uuid import click @@ -39,8 +41,19 @@ def process_app_deployment_request( dns_suffix, deployment_parent_dir, kube_config, - image_registry + image_registry, + log_parent_dir ): + run_id = f"{app_deployment_request.id}-{str(time.time()).split('.')[0]}-{str(uuid.uuid4()).split('-')[0]}" + log_file = None + if log_parent_dir: + log_dir = os.path.join(log_parent_dir, app_deployment_request.id) + if not os.path.exists(log_dir): + os.mkdir(log_dir) + log_file_path = os.path.join(log_dir, f"{run_id}.log") + print(f"Directing build logs to: {log_file_path}") + log_file = open(log_file_path, "wt") + # 1. look up application app = laconic.get_record(app_deployment_request.attributes.application, require=True) @@ -102,8 +115,10 @@ def process_app_deployment_request( needs_k8s_deploy = False # 6. build container (if needed) if not deployment_record or deployment_record.attributes.application != app.id: - build_container_image(app, deployment_container_tag) - push_container_image(deployment_dir) + # TODO: pull from request + extra_build_args = [] + build_container_image(app, deployment_container_tag, extra_build_args, log_file) + push_container_image(deployment_dir, log_file) needs_k8s_deploy = True # 7. update config (if needed) @@ -116,6 +131,7 @@ def process_app_deployment_request( deploy_to_k8s( deployment_record, deployment_dir, + log_file ) publish_deployment( @@ -162,10 +178,14 @@ def dump_known_requests(filename, requests, status="SEEN"): @click.option("--record-namespace-dns", help="eg, crn://laconic/dns") @click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments") @click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True) +@click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="") +@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="") +@click.option("--log-dir", help="Output build/deployment logs to directory.", default=None) @click.pass_context -def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, +def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, # noqa: C901 request_id, discover, state_file, only_update_state, - dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run): + dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run, + include_tags, exclude_tags, log_dir): if request_id and discover: print("Cannot specify both --request-id and --discover", file=sys.stderr) sys.exit(2) @@ -183,6 +203,10 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr) sys.exit(2) + # Split CSV and clean up values. + include_tags = [tag.strip() for tag in include_tags.split(",") if tag] + exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag] + laconic = LaconicRegistryClient(laconic_config) # Find deployment requests. @@ -198,12 +222,24 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ dump_known_requests(state_file, requests) return + def skip_by_tag(r): + for tag in exclude_tags: + if tag and r.attributes.tags and tag in r.attributes.tags: + return True + + for tag in include_tags: + if tag and (not r.attributes.tags or tag not in r.attributes.tags): + return True + + return False + previous_requests = load_known_requests(state_file) # Collapse related requests. requests.sort(key=lambda r: r.createTime) requests.reverse() requests_by_name = {} + skipped_by_name = {} for r in requests: # TODO: Do this _after_ filtering deployments and cancellations to minimize round trips. app = laconic.get_record(r.attributes.application) @@ -216,17 +252,20 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ requested_name = generate_hostname_for_app(app) print("Generating name %s for request %s." % (requested_name, r.id)) - if requested_name not in requests_by_name: - print( - "Found request %s to run application %s on %s." - % (r.id, r.attributes.application, requested_name) - ) - requests_by_name[requested_name] = r - else: - print( - "Ignoring request %s, it is superseded by %s." - % (r.id, requests_by_name[requested_name].id) - ) + if requested_name in skipped_by_name or requested_name in requests_by_name: + print("Ignoring request %s, it has been superseded." % r.id) + continue + + if skip_by_tag(r): + print("Skipping request %s, filtered by tag (include %s, exclude %s, present %s)" % (r.id, + include_tags, + exclude_tags, + r.attributes.tags)) + skipped_by_name[requested_name] = r + continue + + print("Found request %s to run application %s on %s." % (r.id, r.attributes.application, requested_name)) + requests_by_name[requested_name] = r # Find deployments. deployments = laconic.app_deployments() @@ -273,7 +312,8 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ dns_suffix, os.path.abspath(deployment_parent_dir), kube_config, - image_registry + image_registry, + log_dir ) status = "DEPLOYED" finally: diff --git a/stack_orchestrator/deploy/webapp/util.py b/stack_orchestrator/deploy/webapp/util.py index 50485110..74c8cd04 100644 --- a/stack_orchestrator/deploy/webapp/util.py +++ b/stack_orchestrator/deploy/webapp/util.py @@ -212,7 +212,7 @@ def determine_base_container(clone_dir, app_type="webapp"): return base_container -def build_container_image(app_record, tag, extra_build_args=[]): +def build_container_image(app_record, tag, extra_build_args=[], log_file=None): tmpdir = tempfile.mkdtemp() try: @@ -227,10 +227,10 @@ def build_container_image(app_record, tag, extra_build_args=[]): git_env = dict(os.environ.copy()) # Never prompt git_env["GIT_TERMINAL_PROMPT"] = "0" - subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env) - subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env) + subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env, stdout=log_file, stderr=log_file) + subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env, stdout=log_file, stderr=log_file) else: - result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir]) + result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir], stdout=log_file, stderr=log_file) result.check_returncode() base_container = determine_base_container(clone_dir, app_record.attributes.app_type) @@ -246,25 +246,27 @@ def build_container_image(app_record, tag, extra_build_args=[]): build_command.append("--extra-build-args") build_command.append(" ".join(extra_build_args)) - result = subprocess.run(build_command) + result = subprocess.run(build_command, stdout=log_file, stderr=log_file) result.check_returncode() finally: cmd("rm", "-rf", tmpdir) -def push_container_image(deployment_dir): +def push_container_image(deployment_dir, log_file=None): print("Pushing image ...") - result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"]) + result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"], + stdout=log_file, stderr=log_file) result.check_returncode() -def deploy_to_k8s(deploy_record, deployment_dir): +def deploy_to_k8s(deploy_record, deployment_dir, log_file=None): if not deploy_record: command = "up" else: command = "update" - result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command]) + result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command], + stdout=log_file, stderr=log_file) result.check_returncode() From 4a981d8d2e54b870fbfdbc6f3ef18e45c27a6aad Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 7 Feb 2024 18:52:07 +0000 Subject: [PATCH 21/27] Fix repo URL (#733) Needs a '/' (http) not ':' (ssh). Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/733 Co-authored-by: Thomas E Lackey Co-committed-by: Thomas E Lackey --- .../data/stacks/webapp-deployer-backend/stack.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stack_orchestrator/data/stacks/webapp-deployer-backend/stack.yml b/stack_orchestrator/data/stacks/webapp-deployer-backend/stack.yml index ca61049c..04000a1b 100644 --- a/stack_orchestrator/data/stacks/webapp-deployer-backend/stack.yml +++ b/stack_orchestrator/data/stacks/webapp-deployer-backend/stack.yml @@ -2,10 +2,10 @@ version: "1.0" name: webapp-deployer-backend description: "Deployer for webapps" repos: - - git.vdb.to:telackey/webapp-deployment-status-api + - git.vdb.to/telackey/webapp-deployment-status-api containers: - cerc/webapp-deployer-backend pods: - name: webapp-deployer-backend - repository: git.vdb.to:telackey/webapp-deployment-status-api + repository: git.vdb.to/telackey/webapp-deployment-status-api path: ./ From d2ebb81d773186897abad863917e790dd8d4c69b Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 7 Feb 2024 21:45:16 +0000 Subject: [PATCH 22/27] Tags for undeploy (#734) ``` --include-tags TEXT Only include requests with matching tags (comma-separated). --exclude-tags TEXT Exclude requests with matching tags (comma- separated). ``` Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/734 Co-authored-by: Thomas E Lackey Co-committed-by: Thomas E Lackey --- .../webapp/deploy_webapp_from_registry.py | 15 ++---------- .../webapp/undeploy_webapp_from_registry.py | 24 ++++++++++++++++--- stack_orchestrator/deploy/webapp/util.py | 12 ++++++++++ 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 4540ceac..299eeba9 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -29,7 +29,7 @@ from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, build_container_image, push_container_image, file_hash, deploy_to_k8s, publish_deployment, hostname_for_deployment_request, generate_hostname_for_app, - match_owner) + match_owner, skip_by_tag) def process_app_deployment_request( @@ -222,17 +222,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ dump_known_requests(state_file, requests) return - def skip_by_tag(r): - for tag in exclude_tags: - if tag and r.attributes.tags and tag in r.attributes.tags: - return True - - for tag in include_tags: - if tag and (not r.attributes.tags or tag not in r.attributes.tags): - return True - - return False - previous_requests = load_known_requests(state_file) # Collapse related requests. @@ -256,7 +245,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ print("Ignoring request %s, it has been superseded." % r.id) continue - if skip_by_tag(r): + if skip_by_tag(r, include_tags, exclude_tags): print("Skipping request %s, filtered by tag (include %s, exclude %s, present %s)" % (r.id, include_tags, exclude_tags, diff --git a/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py index 74cf0d60..4aa2307d 100644 --- a/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py @@ -20,7 +20,7 @@ import sys import click -from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient, match_owner +from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient, match_owner, skip_by_tag def process_app_removal_request(ctx, @@ -107,10 +107,12 @@ def dump_known_requests(filename, requests): @click.option("--delete-names/--preserve-names", help="Delete all names associated with removed deployments.", default=True) @click.option("--delete-volumes/--preserve-volumes", default=True, help="delete data volumes") @click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True) +@click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="") +@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="") @click.pass_context def command(ctx, laconic_config, deployment_parent_dir, request_id, discover, state_file, only_update_state, - delete_names, delete_volumes, dry_run): + delete_names, delete_volumes, dry_run, include_tags, exclude_tags): if request_id and discover: print("Cannot specify both --request-id and --discover", file=sys.stderr) sys.exit(2) @@ -123,6 +125,10 @@ def command(ctx, laconic_config, deployment_parent_dir, print("--only-update-state requires --state-file", file=sys.stderr) sys.exit(2) + # Split CSV and clean up values. + include_tags = [tag.strip() for tag in include_tags.split(",") if tag] + exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag] + laconic = LaconicRegistryClient(laconic_config) # Find deployment removal requests. @@ -155,10 +161,22 @@ def command(ctx, laconic_config, deployment_parent_dir, # TODO: should we handle CRNs? removals_by_deployment[r.attributes.deployment] = r - requests_to_execute = [] + one_per_deployment = {} for r in requests: if not r.attributes.deployment: print(f"Skipping removal request {r.id} since it was a cancellation.") + elif r.attributes.deployment in one_per_deployment: + print(f"Skipping removal request {r.id} since it was superseded.") + else: + one_per_deployment[r.attributes.deployment] = r + + requests_to_execute = [] + for r in one_per_deployment.values(): + if skip_by_tag(r, include_tags, exclude_tags): + print("Skipping removal request %s, filtered by tag (include %s, exclude %s, present %s)" % (r.id, + include_tags, + exclude_tags, + r.attributes.tags)) elif r.id in removals_by_request: print(f"Found satisfied request for {r.id} at {removals_by_request[r.id].id}") elif r.attributes.deployment in removals_by_deployment: diff --git a/stack_orchestrator/deploy/webapp/util.py b/stack_orchestrator/deploy/webapp/util.py index 74c8cd04..6cdec448 100644 --- a/stack_orchestrator/deploy/webapp/util.py +++ b/stack_orchestrator/deploy/webapp/util.py @@ -351,3 +351,15 @@ def generate_hostname_for_app(app): else: m.update(app.attributes.repository.encode()) return "%s-%s" % (last_part, m.hexdigest()[0:10]) + + +def skip_by_tag(r, include_tags, exclude_tags): + for tag in exclude_tags: + if tag and r.attributes.tags and tag in r.attributes.tags: + return True + + for tag in include_tags: + if tag and (not r.attributes.tags or tag not in r.attributes.tags): + return True + + return False From 8964e1c0fea2bbda79a0bc9457d6076d4778dc13 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 7 Feb 2024 16:48:02 -0600 Subject: [PATCH 23/27] Add resource limit options to spec. --- stack_orchestrator/deploy/k8s/cluster_info.py | 28 ++++++++++++++++--- stack_orchestrator/deploy/spec.py | 6 ++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 35b2b9da..527fe7c5 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -28,6 +28,15 @@ from stack_orchestrator.deploy.deploy_types import DeployEnvVars from stack_orchestrator.deploy.spec import Spec from stack_orchestrator.deploy.images import remote_tag_for_image +DEFAULT_VOLUME_RESOURCES = { + "requests": {"storage": "2Gi"} +} + +DEFAULT_CONTAINER_RESOURCES = { + "requests": {"cpu": "100m", "memory": "200Mi"}, + "limits": {"cpu": "1000m", "memory": "2000Mi"}, +} + class ClusterInfo: parsed_pod_yaml_map: Any @@ -135,6 +144,9 @@ class ClusterInfo: result = [] spec_volumes = self.spec.get_volumes() named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) + resources = self.spec.get_volume_resources() + if not resources: + resources = DEFAULT_VOLUME_RESOURCES if opts.o.debug: print(f"Spec Volumes: {spec_volumes}") print(f"Named Volumes: {named_volumes}") @@ -147,7 +159,8 @@ class ClusterInfo: access_modes=["ReadWriteOnce"], storage_class_name="manual", resources=client.V1ResourceRequirements( - requests={"storage": "2Gi"} + requests=resources.get("requests"), + limits=resources.get("limits") ), volume_name=f"{self.app_name}-{volume_name}" ) @@ -192,6 +205,9 @@ class ClusterInfo: result = [] spec_volumes = self.spec.get_volumes() named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) + resources = self.spec.get_volume_resources() + if not resources: + resources = DEFAULT_VOLUME_RESOURCES for volume_name in spec_volumes: if volume_name not in named_volumes: if opts.o.debug: @@ -200,7 +216,7 @@ class ClusterInfo: spec = client.V1PersistentVolumeSpec( storage_class_name="manual", access_modes=["ReadWriteOnce"], - capacity={"storage": "2Gi"}, + capacity=resources.get("requests", DEFAULT_VOLUME_RESOURCES["requests"]), host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name)) ) pv = client.V1PersistentVolume( @@ -214,6 +230,10 @@ class ClusterInfo: # TODO: put things like image pull policy into an object-scope struct def get_deployment(self, image_pull_policy: str = None): containers = [] + resources = self.spec.get_container_resources() + if not resources: + resources = DEFAULT_CONTAINER_RESOURCES + print(resources) for pod_name in self.parsed_pod_yaml_map: pod = self.parsed_pod_yaml_map[pod_name] services = pod["services"] @@ -238,8 +258,8 @@ class ClusterInfo: ports=[client.V1ContainerPort(container_port=port)], volume_mounts=volume_mounts, resources=client.V1ResourceRequirements( - requests={"cpu": "100m", "memory": "200Mi"}, - limits={"cpu": "1000m", "memory": "2000Mi"}, + requests=resources.get("requests"), + limits=resources.get("limits") ), ) containers.append(container) diff --git a/stack_orchestrator/deploy/spec.py b/stack_orchestrator/deploy/spec.py index dd6cd107..e7fa8583 100644 --- a/stack_orchestrator/deploy/spec.py +++ b/stack_orchestrator/deploy/spec.py @@ -47,6 +47,12 @@ class Spec: if self.obj and "configmaps" in self.obj else {}) + def get_container_resources(self): + return self.obj.get("resources", {}).get("containers") + + def get_volume_resources(self): + return self.obj.get("resources", {}).get("volumes") + def get_http_proxy(self): return (self.obj[constants.network_key][constants.http_proxy_key] if self.obj and constants.network_key in self.obj From 2a9955055ce4452c0eba725036e3ec6c2c122043 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 7 Feb 2024 16:56:35 -0600 Subject: [PATCH 24/27] debug --- stack_orchestrator/deploy/k8s/cluster_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 527fe7c5..c65a299d 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -150,6 +150,7 @@ class ClusterInfo: if opts.o.debug: print(f"Spec Volumes: {spec_volumes}") print(f"Named Volumes: {named_volumes}") + print(f"Resources: {resources}") for volume_name in spec_volumes: if volume_name not in named_volumes: if opts.o.debug: @@ -233,7 +234,6 @@ class ClusterInfo: resources = self.spec.get_container_resources() if not resources: resources = DEFAULT_CONTAINER_RESOURCES - print(resources) for pod_name in self.parsed_pod_yaml_map: pod = self.parsed_pod_yaml_map[pod_name] services = pod["services"] From 4b3b3478e7b931aec88e6509e2acdba7b44fc358 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 8 Feb 2024 00:43:41 -0600 Subject: [PATCH 25/27] Switch to Docker-style limits --- requirements.txt | 1 + stack_orchestrator/deploy/k8s/cluster_info.py | 48 +++++++++++------ stack_orchestrator/deploy/spec.py | 54 +++++++++++++++++-- 3 files changed, 83 insertions(+), 20 deletions(-) diff --git a/requirements.txt b/requirements.txt index bbf97b4a..f6e3d07c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ pydantic==1.10.9 tomli==2.0.1 validators==0.22.0 kubernetes>=28.1.0 +humanfriendly>=10.0 diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index c65a299d..6f7d0cec 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -25,17 +25,37 @@ from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment from stack_orchestrator.deploy.deploy_types import DeployEnvVars -from stack_orchestrator.deploy.spec import Spec +from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits from stack_orchestrator.deploy.images import remote_tag_for_image -DEFAULT_VOLUME_RESOURCES = { - "requests": {"storage": "2Gi"} -} +DEFAULT_VOLUME_RESOURCES = Resources({ + "reservations": {"storage": "2Gi"} +}) -DEFAULT_CONTAINER_RESOURCES = { - "requests": {"cpu": "100m", "memory": "200Mi"}, - "limits": {"cpu": "1000m", "memory": "2000Mi"}, -} +DEFAULT_CONTAINER_RESOURCES = Resources({ + "reservations": {"cpus": "0.1", "memory": "200M"}, + "limits": {"cpus": "1.0", "memory": "2000M"}, +}) + + +def get_k8s_resource_requirements(resources: Resources) -> client.V1ResourceRequirements: + def to_dict(limits: ResourceLimits): + if not limits: + return None + + ret = {} + if limits.cpus: + ret["cpu"] = str(limits.cpus) + if limits.memory: + ret["memory"] = f"{int(limits.memory / (1000 * 1000))}M" + if limits.storage: + ret["storage"] = f"{int(limits.storage / (1000 * 1000))}M" + return ret + + return client.V1ResourceRequirements( + requests=to_dict(resources.reservations), + limits=to_dict(resources.limits) + ) class ClusterInfo: @@ -159,10 +179,7 @@ class ClusterInfo: spec = client.V1PersistentVolumeClaimSpec( access_modes=["ReadWriteOnce"], storage_class_name="manual", - resources=client.V1ResourceRequirements( - requests=resources.get("requests"), - limits=resources.get("limits") - ), + resources=get_k8s_resource_requirements(resources), volume_name=f"{self.app_name}-{volume_name}" ) pvc = client.V1PersistentVolumeClaim( @@ -217,7 +234,7 @@ class ClusterInfo: spec = client.V1PersistentVolumeSpec( storage_class_name="manual", access_modes=["ReadWriteOnce"], - capacity=resources.get("requests", DEFAULT_VOLUME_RESOURCES["requests"]), + capacity=get_k8s_resource_requirements(resources).requests, host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name)) ) pv = client.V1PersistentVolume( @@ -257,10 +274,7 @@ class ClusterInfo: env=envs_from_environment_variables_map(self.environment_variables.map), ports=[client.V1ContainerPort(container_port=port)], volume_mounts=volume_mounts, - resources=client.V1ResourceRequirements( - requests=resources.get("requests"), - limits=resources.get("limits") - ), + resources=get_k8s_resource_requirements(resources), ) containers.append(container) volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec, self.app_name) diff --git a/stack_orchestrator/deploy/spec.py b/stack_orchestrator/deploy/spec.py index e7fa8583..fa0489e7 100644 --- a/stack_orchestrator/deploy/spec.py +++ b/stack_orchestrator/deploy/spec.py @@ -13,12 +13,60 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from pathlib import Path import typing +import humanfriendly + +from pathlib import Path + from stack_orchestrator.util import get_yaml from stack_orchestrator import constants +class ResourceLimits: + cpus: float = None + memory: int = None + storage: int = None + + def __init__(self, obj={}): + if "cpus" in obj: + self.cpus = float(obj["cpus"]) + if "memory" in obj: + self.memory = humanfriendly.parse_size(obj["memory"]) + if "storage" in obj: + self.storage = humanfriendly.parse_size(obj["storage"]) + + def __len__(self): + return len(self.__dict__) + + def __iter__(self): + for k in self.__dict__: + yield k, self.__dict__[k] + + def __repr__(self): + return str(self.__dict__) + + +class Resources: + limits: ResourceLimits = None + reservations: ResourceLimits = None + + def __init__(self, obj={}): + if "reservations" in obj: + self.reservations = ResourceLimits(obj["reservations"]) + if "limits" in obj: + self.limits = ResourceLimits(obj["limits"]) + + def __len__(self): + return len(self.__dict__) + + def __iter__(self): + for k in self.__dict__: + yield k, self.__dict__[k] + + def __repr__(self): + return str(self.__dict__) + + class Spec: obj: typing.Any @@ -48,10 +96,10 @@ class Spec: else {}) def get_container_resources(self): - return self.obj.get("resources", {}).get("containers") + return Resources(self.obj.get("resources", {}).get("containers", {})) def get_volume_resources(self): - return self.obj.get("resources", {}).get("volumes") + return Resources(self.obj.get("resources", {}).get("volumes", {})) def get_http_proxy(self): return (self.obj[constants.network_key][constants.http_proxy_key] From 3309782439373ab252639c0010ae9369de4a8885 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 8 Feb 2024 00:47:46 -0600 Subject: [PATCH 26/27] Refactor --- stack_orchestrator/deploy/k8s/cluster_info.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 6f7d0cec..30b2ab11 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -38,7 +38,7 @@ DEFAULT_CONTAINER_RESOURCES = Resources({ }) -def get_k8s_resource_requirements(resources: Resources) -> client.V1ResourceRequirements: +def to_k8s_resource_requirements(resources: Resources) -> client.V1ResourceRequirements: def to_dict(limits: ResourceLimits): if not limits: return None @@ -179,7 +179,7 @@ class ClusterInfo: spec = client.V1PersistentVolumeClaimSpec( access_modes=["ReadWriteOnce"], storage_class_name="manual", - resources=get_k8s_resource_requirements(resources), + resources=to_k8s_resource_requirements(resources), volume_name=f"{self.app_name}-{volume_name}" ) pvc = client.V1PersistentVolumeClaim( @@ -234,7 +234,7 @@ class ClusterInfo: spec = client.V1PersistentVolumeSpec( storage_class_name="manual", access_modes=["ReadWriteOnce"], - capacity=get_k8s_resource_requirements(resources).requests, + capacity=to_k8s_resource_requirements(resources).requests, host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name)) ) pv = client.V1PersistentVolume( @@ -274,7 +274,7 @@ class ClusterInfo: env=envs_from_environment_variables_map(self.environment_variables.map), ports=[client.V1ContainerPort(container_port=port)], volume_mounts=volume_mounts, - resources=get_k8s_resource_requirements(resources), + resources=to_k8s_resource_requirements(resources), ) containers.append(container) volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec, self.app_name) From 8be1e684e8422d16c128362b960f159b84a5a330 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 8 Feb 2024 19:41:57 +0000 Subject: [PATCH 27/27] Process environment variables defined in compose files (#736) Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/736 Co-authored-by: David Boreham Co-committed-by: David Boreham --- .../data/compose/docker-compose-test.yml | 1 + .../cerc-test-container/run.sh | 3 ++ stack_orchestrator/deploy/k8s/cluster_info.py | 11 ++++- stack_orchestrator/deploy/k8s/helpers.py | 28 +++++++++++++ tests/deploy/run-deploy-test.sh | 41 +++++++++++++++++++ tests/k8s-deploy/run-deploy-test.sh | 9 ++++ 6 files changed, 91 insertions(+), 2 deletions(-) diff --git a/stack_orchestrator/data/compose/docker-compose-test.yml b/stack_orchestrator/data/compose/docker-compose-test.yml index 50151f65..5c9e7f9b 100644 --- a/stack_orchestrator/data/compose/docker-compose-test.yml +++ b/stack_orchestrator/data/compose/docker-compose-test.yml @@ -5,6 +5,7 @@ services: environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_TEST_PARAM_1: ${CERC_TEST_PARAM_1:-FAILED} + CERC_TEST_PARAM_2: "CERC_TEST_PARAM_2_VALUE" volumes: - test-data:/data - test-config:/config:ro diff --git a/stack_orchestrator/data/container-build/cerc-test-container/run.sh b/stack_orchestrator/data/container-build/cerc-test-container/run.sh index 01fb874b..fa860951 100755 --- a/stack_orchestrator/data/container-build/cerc-test-container/run.sh +++ b/stack_orchestrator/data/container-build/cerc-test-container/run.sh @@ -17,6 +17,9 @@ fi if [ -n "$CERC_TEST_PARAM_1" ]; then echo "Test-param-1: ${CERC_TEST_PARAM_1}" fi +if [ -n "$CERC_TEST_PARAM_2" ]; then + echo "Test-param-2: ${CERC_TEST_PARAM_2}" +fi if [ -d "/config" ]; then echo "/config: EXISTS" diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 30b2ab11..17f75651 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -22,7 +22,7 @@ from stack_orchestrator.opts import opts from stack_orchestrator.util import env_var_map_from_file from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path -from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map +from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map, envs_from_compose_file, merge_envs from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment from stack_orchestrator.deploy.deploy_types import DeployEnvVars from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits @@ -263,6 +263,13 @@ class ClusterInfo: if opts.o.debug: print(f"image: {image}") print(f"service port: {port}") + merged_envs = merge_envs( + envs_from_compose_file( + service_info["environment"]), self.environment_variables.map + ) if "environment" in service_info else self.environment_variables.map + envs = envs_from_environment_variables_map(merged_envs) + if opts.o.debug: + print(f"Merged envs: {envs}") # Re-write the image tag for remote deployment image_to_use = remote_tag_for_image( image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image @@ -271,7 +278,7 @@ class ClusterInfo: name=container_name, image=image_to_use, image_pull_policy=image_pull_policy, - env=envs_from_environment_variables_map(self.environment_variables.map), + env=envs, ports=[client.V1ContainerPort(container_port=port)], volume_mounts=volume_mounts, resources=to_k8s_resource_requirements(resources), diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index e386b353..1fe697e7 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -17,6 +17,7 @@ from kubernetes import client import os from pathlib import Path import subprocess +import re from typing import Set, Mapping, List from stack_orchestrator.opts import opts @@ -214,6 +215,33 @@ def _generate_kind_port_mappings(parsed_pod_files): ) +# Note: this makes any duplicate definition in b overwrite a +def merge_envs(a: Mapping[str, str], b: Mapping[str, str]) -> Mapping[str, str]: + result = {**a, **b} + return result + + +def _expand_shell_vars(raw_val: str) -> str: + # could be: or ${} or ${:-} + # TODO: implement support for variable substitution and default values + # if raw_val is like ${} print a warning and substitute an empty string + # otherwise return raw_val + match = re.search(r"^\$\{(.*)\}$", raw_val) + if match: + print(f"WARNING: found unimplemented environment variable substitution: {raw_val}") + else: + return raw_val + + +# TODO: handle the case where the same env var is defined in multiple places +def envs_from_compose_file(compose_file_envs: Mapping[str, str]) -> Mapping[str, str]: + result = {} + for env_var, env_val in compose_file_envs.items(): + expanded_env_val = _expand_shell_vars(env_val) + result.update({env_var: expanded_env_val}) + return result + + def envs_from_environment_variables_map(map: Mapping[str, str]) -> List[client.V1EnvVar]: result = [] for env_var, env_val in map.items(): diff --git a/tests/deploy/run-deploy-test.sh b/tests/deploy/run-deploy-test.sh index 4456c28d..f7a008f3 100755 --- a/tests/deploy/run-deploy-test.sh +++ b/tests/deploy/run-deploy-test.sh @@ -6,6 +6,12 @@ fi # Dump environment variables for debugging echo "Environment variables:" env + +delete_cluster_exit () { + $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes + exit 1 +} + # Test basic stack-orchestrator deploy echo "Running stack-orchestrator deploy test" # Bit of a hack, test the most recent package @@ -106,6 +112,10 @@ if [ ! "$create_file_content" == "create-command-output-data" ]; then echo "deploy create test: FAILED" exit 1 fi + +# Add a config file to be picked up by the ConfigMap before starting. +echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/data/test-config/test_config + echo "deploy create output file test: passed" # Try to start the deployment $TEST_TARGET_SO deployment --dir $test_deployment_dir start @@ -124,6 +134,37 @@ else echo "deployment config test: FAILED" exit 1 fi +# Check the config variable CERC_TEST_PARAM_2 was passed correctly from the compose file +if [[ "$log_output_3" == *"Test-param-2: CERC_TEST_PARAM_2_VALUE"* ]]; then + echo "deployment compose config test: passed" +else + echo "deployment compose config test: FAILED" + exit 1 +fi + +# Check that the ConfigMap is mounted and contains the expected content. +log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) +if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then + echo "deployment ConfigMap test: passed" +else + echo "deployment ConfigMap test: FAILED" + delete_cluster_exit +fi + +# Stop then start again and check the volume was preserved +$TEST_TARGET_SO deployment --dir $test_deployment_dir stop +# Sleep a bit just in case +# sleep for longer to check if that's why the subsequent create cluster fails +sleep 20 +$TEST_TARGET_SO deployment --dir $test_deployment_dir start +log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) +if [[ "$log_output_5" == *"Filesystem is old"* ]]; then + echo "Retain volumes test: passed" +else + echo "Retain volumes test: FAILED" + delete_cluster_exit +fi + # Stop and clean up $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes echo "Test passed" diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh index 04008217..4b17ae49 100755 --- a/tests/k8s-deploy/run-deploy-test.sh +++ b/tests/k8s-deploy/run-deploy-test.sh @@ -114,6 +114,7 @@ else echo "deployment logs test: FAILED" delete_cluster_exit fi + # Check the config variable CERC_TEST_PARAM_1 was passed correctly if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then echo "deployment config test: passed" @@ -122,6 +123,14 @@ else delete_cluster_exit fi +# Check the config variable CERC_TEST_PARAM_2 was passed correctly from the compose file +if [[ "$log_output_3" == *"Test-param-2: CERC_TEST_PARAM_2_VALUE"* ]]; then + echo "deployment compose config test: passed" +else + echo "deployment compose config test: FAILED" + exit 1 +fi + # Check that the ConfigMap is mounted and contains the expected content. log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then