Check for existing tag in remote repo before building. #764

Merged
telackey merged 14 commits from telackey/remotetag into main 2024-02-24 03:22:51 +00:00
3 changed files with 53 additions and 10 deletions

View File

@ -21,6 +21,8 @@
# TODO: display the available list of containers; allow re-build of either all or specific containers # TODO: display the available list of containers; allow re-build of either all or specific containers
import os import os
import sys
from decouple import config from decouple import config
import click import click
from pathlib import Path from pathlib import Path
@ -40,12 +42,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
'''build the specified webapp container''' '''build the specified webapp container'''
quiet = ctx.obj.quiet quiet = ctx.obj.quiet
verbose = ctx.obj.verbose
dry_run = ctx.obj.dry_run
debug = ctx.obj.debug debug = ctx.obj.debug
local_stack = ctx.obj.local_stack local_stack = ctx.obj.local_stack
stack = ctx.obj.stack stack = ctx.obj.stack
continue_on_error = ctx.obj.continue_on_error
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build") container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
@ -73,7 +72,10 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
container_build_env, container_build_env,
dev_root_path, dev_root_path,
) )
build_containers.process_container(build_context_1) ok = build_containers.process_container(build_context_1)
if not ok:
print("ERROR: Build failed.", file=sys.stderr)
sys.exit(1)
# Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir. # Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir.
container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true" container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
@ -94,4 +96,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
container_build_env, container_build_env,
dev_root_path, dev_root_path,
) )
build_containers.process_container(build_context_2) ok = build_containers.process_container(build_context_2)
if not ok:
print("ERROR: Build failed.", file=sys.stderr)
sys.exit(1)

View File

@ -29,6 +29,29 @@ def _image_needs_pushed(image: str):
return image.endswith(":local") return image.endswith(":local")
def remote_image_exists(remote_repo_url: str, local_tag: str):
docker = DockerClient()
try:
remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
result = docker.manifest.inspect(remote_tag)
return True if result else False
except Exception: # noqa: E722
return False
def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
if not additional_tags:
return
if not remote_image_exists(remote_repo_url, local_tag):
raise Exception(f"{local_tag} does not exist in {remote_repo_url}")
docker = DockerClient()
remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags)
def remote_tag_for_image(image: str, remote_repo_url: str): def remote_tag_for_image(image: str, remote_repo_url: str):
# Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
major_parts = image.split("/", 2) major_parts = image.split("/", 2)

View File

@ -24,6 +24,7 @@ import uuid
import click import click
from stack_orchestrator.deploy.images import remote_image_exists, add_tags_to_image
from stack_orchestrator.deploy.webapp import deploy_webapp from stack_orchestrator.deploy.webapp import deploy_webapp
from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
build_container_image, push_container_image, build_container_image, push_container_image,
@ -43,6 +44,7 @@ def process_app_deployment_request(
deployment_parent_dir, deployment_parent_dir,
kube_config, kube_config,
image_registry, image_registry,
force_rebuild=False,
log_file=None log_file=None
): ):
# 1. look up application # 1. look up application
@ -91,7 +93,9 @@ def process_app_deployment_request(
deployment_record = laconic.get_record(app_deployment_crn) deployment_record = laconic.get_record(app_deployment_crn)
deployment_dir = os.path.join(deployment_parent_dir, fqdn) deployment_dir = os.path.join(deployment_parent_dir, fqdn)
deployment_config_file = os.path.join(deployment_dir, "config.env") deployment_config_file = os.path.join(deployment_dir, "config.env")
# TODO: Is there any reason not to simplify the hash input to the app_deployment_crn?
deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest() deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest()
app_image_shared_tag = f"laconic-webapp/{app.id}:local"
# b. check for deployment directory (create if necessary) # b. check for deployment directory (create if necessary)
if not os.path.exists(deployment_dir): if not os.path.exists(deployment_dir):
if deployment_record: if deployment_record:
@ -106,11 +110,20 @@ def process_app_deployment_request(
needs_k8s_deploy = False needs_k8s_deploy = False
# 6. build container (if needed) # 6. build container (if needed)
if not deployment_record or deployment_record.attributes.application != app.id: if not deployment_record or deployment_record.attributes.application != app.id:
# TODO: pull from request needs_k8s_deploy = True
extra_build_args = [] # check if the image already exists
shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag)
if shared_tag_exists and not force_rebuild:
# simply add our unique tag to the existing image and we are done
print(f"Using existing app image {app_image_shared_tag} for {deployment_container_tag}", file=log_file)
add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
else:
extra_build_args = [] # TODO: pull from request
build_container_image(app, deployment_container_tag, extra_build_args, log_file) build_container_image(app, deployment_container_tag, extra_build_args, log_file)
push_container_image(deployment_dir, log_file) push_container_image(deployment_dir, log_file)
needs_k8s_deploy = True # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
print(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_tag}", file=log_file)
add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
# 7. update config (if needed) # 7. update config (if needed)
if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config: if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config:
@ -171,12 +184,13 @@ def dump_known_requests(filename, requests, status="SEEN"):
@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True) @click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
@click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="") @click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="")
@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="") @click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="")
@click.option("--force-rebuild", help="Rebuild even if the image already exists.", is_flag=True)
@click.option("--log-dir", help="Output build/deployment logs to directory.", default=None) @click.option("--log-dir", help="Output build/deployment logs to directory.", default=None)
@click.pass_context @click.pass_context
def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, # noqa: C901 def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, # noqa: C901
request_id, discover, state_file, only_update_state, request_id, discover, state_file, only_update_state,
dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run, dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run,
include_tags, exclude_tags, log_dir): include_tags, exclude_tags, force_rebuild, log_dir):
if request_id and discover: if request_id and discover:
print("Cannot specify both --request-id and --discover", file=sys.stderr) print("Cannot specify both --request-id and --discover", file=sys.stderr)
sys.exit(2) sys.exit(2)
@ -306,6 +320,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
os.path.abspath(deployment_parent_dir), os.path.abspath(deployment_parent_dir),
kube_config, kube_config,
image_registry, image_registry,
force_rebuild,
run_log_file run_log_file
) )
status = "DEPLOYED" status = "DEPLOYED"