From 2d8cc09e6fee9c422a1e448abb0facab8489ff07 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 23 Feb 2024 14:37:25 -0600 Subject: [PATCH 01/13] WIP: Check for existing tag in remote repo before building. --- stack_orchestrator/deploy/images.py | 9 +++++++ .../webapp/deploy_webapp_from_registry.py | 24 ++++++++++++------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py index 7ddcca33..8bfcc794 100644 --- a/stack_orchestrator/deploy/images.py +++ b/stack_orchestrator/deploy/images.py @@ -29,6 +29,15 @@ def _image_needs_pushed(image: str): return image.endswith(":local") +def remote_image_exists(local_tag: str, remote_repo_url: str): + docker = DockerClient() + try: + result = docker.manifest.inspect(remote_tag_for_image(local_tag, remote_repo_url)) + return True if result else False + except: + return False + + def remote_tag_for_image(image: str, remote_repo_url: str): # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy major_parts = image.split("/", 2) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 15e89d9e..e20d0e2d 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -24,6 +24,8 @@ import uuid import click +from stack_orchestrator import constants +from stack_orchestrator.deploy.images import remote_image_exists from stack_orchestrator.deploy.webapp import deploy_webapp from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, build_container_image, push_container_image, @@ -43,7 +45,8 @@ def process_app_deployment_request( deployment_parent_dir, kube_config, image_registry, - log_file=None + force_rebuild = False, + log_file = None ): # 1. look up application app = laconic.get_record(app_deployment_request.attributes.application, require=True) @@ -91,7 +94,7 @@ def process_app_deployment_request( deployment_record = laconic.get_record(app_deployment_crn) deployment_dir = os.path.join(deployment_parent_dir, fqdn) deployment_config_file = os.path.join(deployment_dir, "config.env") - deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest() + deployment_container_tag = f"laconic-webapp/{app.id}:local" # b. check for deployment directory (create if necessary) if not os.path.exists(deployment_dir): if deployment_record: @@ -106,11 +109,14 @@ def process_app_deployment_request( needs_k8s_deploy = False # 6. build container (if needed) if not deployment_record or deployment_record.attributes.application != app.id: - # TODO: pull from request - extra_build_args = [] - build_container_image(app, deployment_container_tag, extra_build_args, log_file) - push_container_image(deployment_dir, log_file) - needs_k8s_deploy = True + print(ctx) + # check if the image already exists + if force_rebuild or not remote_image_exists(deployment_container_tag, ctx.spec[constants.image_registry_key]): + # TODO: pull from request + extra_build_args = [] + build_container_image(app, deployment_container_tag, extra_build_args, log_file) + push_container_image(deployment_dir, log_file) + needs_k8s_deploy = True # 7. update config (if needed) if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config: @@ -171,12 +177,13 @@ def dump_known_requests(filename, requests, status="SEEN"): @click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True) @click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="") @click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="") +@click.option("--force-rebuild", help="Rebuild even if the image already exists.", is_flag=True) @click.option("--log-dir", help="Output build/deployment logs to directory.", default=None) @click.pass_context def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, # noqa: C901 request_id, discover, state_file, only_update_state, dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run, - include_tags, exclude_tags, log_dir): + include_tags, exclude_tags, force_rebuild, log_dir): if request_id and discover: print("Cannot specify both --request-id and --discover", file=sys.stderr) sys.exit(2) @@ -306,6 +313,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ os.path.abspath(deployment_parent_dir), kube_config, image_registry, + force_rebuild, run_log_file ) status = "DEPLOYED" -- 2.45.2 From 2a34f436783a7f9f444998a39a41907a935ff109 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 23 Feb 2024 15:22:15 -0600 Subject: [PATCH 02/13] refactor --- stack_orchestrator/build/build_webapp.py | 15 ++++++++++----- stack_orchestrator/deploy/images.py | 2 +- .../deploy/webapp/deploy_webapp_from_registry.py | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/stack_orchestrator/build/build_webapp.py b/stack_orchestrator/build/build_webapp.py index fcdd8f66..25fbc850 100644 --- a/stack_orchestrator/build/build_webapp.py +++ b/stack_orchestrator/build/build_webapp.py @@ -21,6 +21,8 @@ # TODO: display the available list of containers; allow re-build of either all or specific containers import os +import sys + from decouple import config import click from pathlib import Path @@ -40,12 +42,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t '''build the specified webapp container''' quiet = ctx.obj.quiet - verbose = ctx.obj.verbose - dry_run = ctx.obj.dry_run debug = ctx.obj.debug local_stack = ctx.obj.local_stack stack = ctx.obj.stack - continue_on_error = ctx.obj.continue_on_error # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build") @@ -73,7 +72,10 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t container_build_env, dev_root_path, ) - build_containers.process_container(build_context_1) + ok = build_containers.process_container(build_context_1) + if not ok: + print("ERROR: Build failed.", file=sys.stderr) + sys.exit(1) # Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir. container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true" @@ -94,4 +96,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t container_build_env, dev_root_path, ) - build_containers.process_container(build_context_2) + ok = build_containers.process_container(build_context_2) + if not ok: + print("ERROR: Build failed.", file=sys.stderr) + sys.exit(1) \ No newline at end of file diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py index 8bfcc794..d4a2d6b4 100644 --- a/stack_orchestrator/deploy/images.py +++ b/stack_orchestrator/deploy/images.py @@ -34,7 +34,7 @@ def remote_image_exists(local_tag: str, remote_repo_url: str): try: result = docker.manifest.inspect(remote_tag_for_image(local_tag, remote_repo_url)) return True if result else False - except: + except: # noqa: E722 return False diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index e20d0e2d..51bd32a3 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -45,8 +45,8 @@ def process_app_deployment_request( deployment_parent_dir, kube_config, image_registry, - force_rebuild = False, - log_file = None + force_rebuild=False, + log_file=None ): # 1. look up application app = laconic.get_record(app_deployment_request.attributes.application, require=True) -- 2.45.2 From a408d2490559e322e67b4c8d07d160e7627544e7 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 23 Feb 2024 18:01:44 -0600 Subject: [PATCH 03/13] g --- stack_orchestrator/deploy/images.py | 8 ++++++-- .../deploy/webapp/deploy_webapp_from_registry.py | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py index d4a2d6b4..bf322656 100644 --- a/stack_orchestrator/deploy/images.py +++ b/stack_orchestrator/deploy/images.py @@ -32,9 +32,13 @@ def _image_needs_pushed(image: str): def remote_image_exists(local_tag: str, remote_repo_url: str): docker = DockerClient() try: - result = docker.manifest.inspect(remote_tag_for_image(local_tag, remote_repo_url)) + remote_tag = remote_tag_for_image(local_tag, remote_repo_url) + print(f"local tag: {local_tag} ; remote tag: {remote_tag}") + result = docker.manifest.inspect(remote_tag) + print(result) return True if result else False - except: # noqa: E722 + except Exception as e: # noqa: E722 + print(e) return False diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 51bd32a3..63437c17 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -26,6 +26,7 @@ import click from stack_orchestrator import constants from stack_orchestrator.deploy.images import remote_image_exists +from stack_orchestrator.deploy.spec import Spec from stack_orchestrator.deploy.webapp import deploy_webapp from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, build_container_image, push_container_image, @@ -109,9 +110,8 @@ def process_app_deployment_request( needs_k8s_deploy = False # 6. build container (if needed) if not deployment_record or deployment_record.attributes.application != app.id: - print(ctx) # check if the image already exists - if force_rebuild or not remote_image_exists(deployment_container_tag, ctx.spec[constants.image_registry_key]): + if force_rebuild or not remote_image_exists(deployment_container_tag, image_registry): # TODO: pull from request extra_build_args = [] build_container_image(app, deployment_container_tag, extra_build_args, log_file) -- 2.45.2 From f06b5ec6a21f48bf988ec57d9240d23df3c4ba04 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 23 Feb 2024 18:02:11 -0600 Subject: [PATCH 04/13] Remote tag check --- stack_orchestrator/deploy/images.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py index bf322656..78fe6204 100644 --- a/stack_orchestrator/deploy/images.py +++ b/stack_orchestrator/deploy/images.py @@ -33,12 +33,9 @@ def remote_image_exists(local_tag: str, remote_repo_url: str): docker = DockerClient() try: remote_tag = remote_tag_for_image(local_tag, remote_repo_url) - print(f"local tag: {local_tag} ; remote tag: {remote_tag}") result = docker.manifest.inspect(remote_tag) - print(result) return True if result else False except Exception as e: # noqa: E722 - print(e) return False -- 2.45.2 From d8f5d41261cc1976a661117b82c4827fc307b74f Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 23 Feb 2024 18:06:09 -0600 Subject: [PATCH 05/13] lint --- stack_orchestrator/deploy/images.py | 2 +- .../deploy/webapp/deploy_webapp_from_registry.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py index 78fe6204..78ce99db 100644 --- a/stack_orchestrator/deploy/images.py +++ b/stack_orchestrator/deploy/images.py @@ -35,7 +35,7 @@ def remote_image_exists(local_tag: str, remote_repo_url: str): remote_tag = remote_tag_for_image(local_tag, remote_repo_url) result = docker.manifest.inspect(remote_tag) return True if result else False - except Exception as e: # noqa: E722 + except Exception: # noqa: E722 return False diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 63437c17..aa40e792 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -12,7 +12,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import hashlib import json import os import shlex @@ -24,9 +23,7 @@ import uuid import click -from stack_orchestrator import constants from stack_orchestrator.deploy.images import remote_image_exists -from stack_orchestrator.deploy.spec import Spec from stack_orchestrator.deploy.webapp import deploy_webapp from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, build_container_image, push_container_image, -- 2.45.2 From b0d003cc017d4a20ad3d24fb46174cb1525de5b2 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 23 Feb 2024 18:52:37 -0600 Subject: [PATCH 06/13] Working --- stack_orchestrator/deploy/images.py | 15 ++++++++++++- .../webapp/deploy_webapp_from_registry.py | 22 +++++++++++++------ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py index 78ce99db..77713d18 100644 --- a/stack_orchestrator/deploy/images.py +++ b/stack_orchestrator/deploy/images.py @@ -29,7 +29,7 @@ def _image_needs_pushed(image: str): return image.endswith(":local") -def remote_image_exists(local_tag: str, remote_repo_url: str): +def remote_image_exists(remote_repo_url: str, local_tag: str): docker = DockerClient() try: remote_tag = remote_tag_for_image(local_tag, remote_repo_url) @@ -39,6 +39,19 @@ def remote_image_exists(local_tag: str, remote_repo_url: str): return False +def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags): + if not additional_tags: + return + + if not remote_image_exists(remote_repo_url, local_tag): + raise Exception(f"{local_tag} does not exist in {remote_repo_url}") + + docker = DockerClient() + remote_tag = remote_tag_for_image(local_tag, remote_repo_url) + new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags] + docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags) + + def remote_tag_for_image(image: str, remote_repo_url: str): # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy major_parts = image.split("/", 2) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index aa40e792..3178a668 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -12,6 +12,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import hashlib import json import os import shlex @@ -23,7 +24,7 @@ import uuid import click -from stack_orchestrator.deploy.images import remote_image_exists +from stack_orchestrator.deploy.images import remote_image_exists, add_tags_to_image from stack_orchestrator.deploy.webapp import deploy_webapp from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, build_container_image, push_container_image, @@ -92,34 +93,41 @@ def process_app_deployment_request( deployment_record = laconic.get_record(app_deployment_crn) deployment_dir = os.path.join(deployment_parent_dir, fqdn) deployment_config_file = os.path.join(deployment_dir, "config.env") - deployment_container_tag = f"laconic-webapp/{app.id}:local" + deployment_container_unique_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest() + deployment_container_shared_tag = f"laconic-webapp/{app.id}:local" # b. check for deployment directory (create if necessary) if not os.path.exists(deployment_dir): if deployment_record: raise Exception("Deployment record %s exists, but not deployment dir %s. Please remove name." % (app_deployment_crn, deployment_dir)) print("deploy_webapp", deployment_dir) - deploy_webapp.create_deployment(ctx, deployment_dir, deployment_container_tag, + deploy_webapp.create_deployment(ctx, deployment_dir, deployment_container_unique_tag, f"https://{fqdn}", kube_config, image_registry, env_filename) elif env_filename: shutil.copyfile(env_filename, deployment_config_file) needs_k8s_deploy = False + built_images = False # 6. build container (if needed) if not deployment_record or deployment_record.attributes.application != app.id: # check if the image already exists - if force_rebuild or not remote_image_exists(deployment_container_tag, image_registry): + if force_rebuild or not remote_image_exists(image_registry, deployment_container_shared_tag): # TODO: pull from request extra_build_args = [] - build_container_image(app, deployment_container_tag, extra_build_args, log_file) + build_container_image(app, deployment_container_shared_tag, extra_build_args, log_file) push_container_image(deployment_dir, log_file) + built_images = True needs_k8s_deploy = True - # 7. update config (if needed) + # 7. Add tags (as needed) + if built_images or not remote_image_exists(image_registry, deployment_container_unique_tag): + add_tags_to_image(image_registry, deployment_container_shared_tag, deployment_container_unique_tag) + + # 8. update config (if needed) if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config: needs_k8s_deploy = True - # 8. update k8s deployment + # 9. update k8s deployment if needs_k8s_deploy: print("Deploying to k8s") deploy_to_k8s( -- 2.45.2 From 61d254cd6c8dba065befe2d3db6167969103d5f9 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 23 Feb 2024 18:59:46 -0600 Subject: [PATCH 07/13] Add comment. --- stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 3178a668..d53ab55d 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -93,6 +93,7 @@ def process_app_deployment_request( deployment_record = laconic.get_record(app_deployment_crn) deployment_dir = os.path.join(deployment_parent_dir, fqdn) deployment_config_file = os.path.join(deployment_dir, "config.env") + # TODO: Is there any reason not to simplify the hash input to the app_deployment_crn? deployment_container_unique_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest() deployment_container_shared_tag = f"laconic-webapp/{app.id}:local" # b. check for deployment directory (create if necessary) -- 2.45.2 From 3194ebf8a6bf06a58ca6f68631ba3ff4fe21e0d8 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 23 Feb 2024 19:41:03 -0600 Subject: [PATCH 08/13] Refactor --- .../deploy/webapp/deploy_webapp_from_registry.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index d53ab55d..de920273 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -95,7 +95,7 @@ def process_app_deployment_request( deployment_config_file = os.path.join(deployment_dir, "config.env") # TODO: Is there any reason not to simplify the hash input to the app_deployment_crn? deployment_container_unique_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest() - deployment_container_shared_tag = f"laconic-webapp/{app.id}:local" + app_image_shared_tag = f"laconic-webapp/{app.id}:local" # b. check for deployment directory (create if necessary) if not os.path.exists(deployment_dir): if deployment_record: @@ -112,17 +112,17 @@ def process_app_deployment_request( # 6. build container (if needed) if not deployment_record or deployment_record.attributes.application != app.id: # check if the image already exists - if force_rebuild or not remote_image_exists(image_registry, deployment_container_shared_tag): + if force_rebuild or not remote_image_exists(image_registry, app_image_shared_tag): # TODO: pull from request extra_build_args = [] - build_container_image(app, deployment_container_shared_tag, extra_build_args, log_file) + build_container_image(app, app_image_shared_tag, extra_build_args, log_file) push_container_image(deployment_dir, log_file) built_images = True needs_k8s_deploy = True # 7. Add tags (as needed) if built_images or not remote_image_exists(image_registry, deployment_container_unique_tag): - add_tags_to_image(image_registry, deployment_container_shared_tag, deployment_container_unique_tag) + add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_unique_tag) # 8. update config (if needed) if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config: -- 2.45.2 From a54e9c43a0d8792a07f87b77620c768ad81d279b Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 23 Feb 2024 19:52:57 -0600 Subject: [PATCH 09/13] push-images will only use the compose-defined tag --- .../webapp/deploy_webapp_from_registry.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index de920273..babfa547 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -108,27 +108,28 @@ def process_app_deployment_request( shutil.copyfile(env_filename, deployment_config_file) needs_k8s_deploy = False - built_images = False # 6. build container (if needed) if not deployment_record or deployment_record.attributes.application != app.id: + needs_k8s_deploy = True + # check if the image already exists - if force_rebuild or not remote_image_exists(image_registry, app_image_shared_tag): + shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag) + if shared_tag_exists and not force_rebuild: + # simply add our unique tag to the existing image and we are done + add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_unique_tag) + else: # TODO: pull from request extra_build_args = [] - build_container_image(app, app_image_shared_tag, extra_build_args, log_file) + build_container_image(app, deployment_container_unique_tag, extra_build_args, log_file) push_container_image(deployment_dir, log_file) - built_images = True - needs_k8s_deploy = True + # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag. + add_tags_to_image(image_registry, deployment_container_unique_tag, app_image_shared_tag) - # 7. Add tags (as needed) - if built_images or not remote_image_exists(image_registry, deployment_container_unique_tag): - add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_unique_tag) - - # 8. update config (if needed) + # 7. update config (if needed) if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config: needs_k8s_deploy = True - # 9. update k8s deployment + # 8. update k8s deployment if needs_k8s_deploy: print("Deploying to k8s") deploy_to_k8s( -- 2.45.2 From 8efe753ca4cb76fa486b60626cabfd0774b99437 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 23 Feb 2024 19:53:37 -0600 Subject: [PATCH 10/13] comment --- .../deploy/webapp/deploy_webapp_from_registry.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index babfa547..23bb0a3f 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -118,8 +118,7 @@ def process_app_deployment_request( # simply add our unique tag to the existing image and we are done add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_unique_tag) else: - # TODO: pull from request - extra_build_args = [] + extra_build_args = [] # TODO: pull from request build_container_image(app, deployment_container_unique_tag, extra_build_args, log_file) push_container_image(deployment_dir, log_file) # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag. -- 2.45.2 From f27248b5089f66ffc4b14a87c9f41dce2388cbad Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 23 Feb 2024 19:54:00 -0600 Subject: [PATCH 11/13] fmt --- stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py | 1 - 1 file changed, 1 deletion(-) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 23bb0a3f..859c6b0b 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -111,7 +111,6 @@ def process_app_deployment_request( # 6. build container (if needed) if not deployment_record or deployment_record.attributes.application != app.id: needs_k8s_deploy = True - # check if the image already exists shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag) if shared_tag_exists and not force_rebuild: -- 2.45.2 From 694590ed565f2c3517cb073264733bd3b99776f1 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 23 Feb 2024 20:54:06 -0600 Subject: [PATCH 12/13] logging --- stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 859c6b0b..65dcb748 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -115,12 +115,14 @@ def process_app_deployment_request( shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag) if shared_tag_exists and not force_rebuild: # simply add our unique tag to the existing image and we are done + print(f"Using existing app image {app_image_shared_tag} for {deployment_container_unique_tag}", file=log_file) add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_unique_tag) else: extra_build_args = [] # TODO: pull from request build_container_image(app, deployment_container_unique_tag, extra_build_args, log_file) push_container_image(deployment_dir, log_file) # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag. + print(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_unique_tag}", file=log_file) add_tags_to_image(image_registry, deployment_container_unique_tag, app_image_shared_tag) # 7. update config (if needed) -- 2.45.2 From 39bf0e2795d5ba8f9f9824f58688d0eb63e58fd6 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 23 Feb 2024 20:58:33 -0600 Subject: [PATCH 13/13] refactor --- .../deploy/webapp/deploy_webapp_from_registry.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 65dcb748..027a76b3 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -94,7 +94,7 @@ def process_app_deployment_request( deployment_dir = os.path.join(deployment_parent_dir, fqdn) deployment_config_file = os.path.join(deployment_dir, "config.env") # TODO: Is there any reason not to simplify the hash input to the app_deployment_crn? - deployment_container_unique_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest() + deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest() app_image_shared_tag = f"laconic-webapp/{app.id}:local" # b. check for deployment directory (create if necessary) if not os.path.exists(deployment_dir): @@ -102,7 +102,7 @@ def process_app_deployment_request( raise Exception("Deployment record %s exists, but not deployment dir %s. Please remove name." % (app_deployment_crn, deployment_dir)) print("deploy_webapp", deployment_dir) - deploy_webapp.create_deployment(ctx, deployment_dir, deployment_container_unique_tag, + deploy_webapp.create_deployment(ctx, deployment_dir, deployment_container_tag, f"https://{fqdn}", kube_config, image_registry, env_filename) elif env_filename: shutil.copyfile(env_filename, deployment_config_file) @@ -115,15 +115,15 @@ def process_app_deployment_request( shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag) if shared_tag_exists and not force_rebuild: # simply add our unique tag to the existing image and we are done - print(f"Using existing app image {app_image_shared_tag} for {deployment_container_unique_tag}", file=log_file) - add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_unique_tag) + print(f"Using existing app image {app_image_shared_tag} for {deployment_container_tag}", file=log_file) + add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag) else: extra_build_args = [] # TODO: pull from request - build_container_image(app, deployment_container_unique_tag, extra_build_args, log_file) + build_container_image(app, deployment_container_tag, extra_build_args, log_file) push_container_image(deployment_dir, log_file) # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag. - print(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_unique_tag}", file=log_file) - add_tags_to_image(image_registry, deployment_container_unique_tag, app_image_shared_tag) + print(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_tag}", file=log_file) + add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag) # 7. update config (if needed) if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config: -- 2.45.2