forked from cerc-io/stack-orchestrator
Check for existing tag in remote repo before building. (#764)
webapps are meant to be build-once/deploy-many, but we were rebuilding them for every request. This changes that, so that we rebuild only for every unique ApplicationRecord. When we push the image, we now tag it according to its ApplicationRecord. We don't want to use that tag directly in the compose file for the deployment, however, as the deployment needs to be able to adjust to new builds w/o re-writing the file all the time. Instead, we use a per-deployment unique tag (same as before), we just update what image it references as needed. Reviewed-on: cerc-io/stack-orchestrator#764
This commit is contained in:
parent
a16fc657bf
commit
a0413659f7
@ -21,6 +21,8 @@
|
||||
# TODO: display the available list of containers; allow re-build of either all or specific containers
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from decouple import config
|
||||
import click
|
||||
from pathlib import Path
|
||||
@ -40,12 +42,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
|
||||
'''build the specified webapp container'''
|
||||
|
||||
quiet = ctx.obj.quiet
|
||||
verbose = ctx.obj.verbose
|
||||
dry_run = ctx.obj.dry_run
|
||||
debug = ctx.obj.debug
|
||||
local_stack = ctx.obj.local_stack
|
||||
stack = ctx.obj.stack
|
||||
continue_on_error = ctx.obj.continue_on_error
|
||||
|
||||
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
|
||||
container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
|
||||
@ -73,7 +72,10 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
|
||||
container_build_env,
|
||||
dev_root_path,
|
||||
)
|
||||
build_containers.process_container(build_context_1)
|
||||
ok = build_containers.process_container(build_context_1)
|
||||
if not ok:
|
||||
print("ERROR: Build failed.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir.
|
||||
container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
|
||||
@ -94,4 +96,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
|
||||
container_build_env,
|
||||
dev_root_path,
|
||||
)
|
||||
build_containers.process_container(build_context_2)
|
||||
ok = build_containers.process_container(build_context_2)
|
||||
if not ok:
|
||||
print("ERROR: Build failed.", file=sys.stderr)
|
||||
sys.exit(1)
|
@ -29,6 +29,29 @@ def _image_needs_pushed(image: str):
|
||||
return image.endswith(":local")
|
||||
|
||||
|
||||
def remote_image_exists(remote_repo_url: str, local_tag: str):
|
||||
docker = DockerClient()
|
||||
try:
|
||||
remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
|
||||
result = docker.manifest.inspect(remote_tag)
|
||||
return True if result else False
|
||||
except Exception: # noqa: E722
|
||||
return False
|
||||
|
||||
|
||||
def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
|
||||
if not additional_tags:
|
||||
return
|
||||
|
||||
if not remote_image_exists(remote_repo_url, local_tag):
|
||||
raise Exception(f"{local_tag} does not exist in {remote_repo_url}")
|
||||
|
||||
docker = DockerClient()
|
||||
remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
|
||||
new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
|
||||
docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags)
|
||||
|
||||
|
||||
def remote_tag_for_image(image: str, remote_repo_url: str):
|
||||
# Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
|
||||
major_parts = image.split("/", 2)
|
||||
|
@ -24,6 +24,7 @@ import uuid
|
||||
|
||||
import click
|
||||
|
||||
from stack_orchestrator.deploy.images import remote_image_exists, add_tags_to_image
|
||||
from stack_orchestrator.deploy.webapp import deploy_webapp
|
||||
from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
|
||||
build_container_image, push_container_image,
|
||||
@ -43,6 +44,7 @@ def process_app_deployment_request(
|
||||
deployment_parent_dir,
|
||||
kube_config,
|
||||
image_registry,
|
||||
force_rebuild=False,
|
||||
log_file=None
|
||||
):
|
||||
# 1. look up application
|
||||
@ -91,7 +93,9 @@ def process_app_deployment_request(
|
||||
deployment_record = laconic.get_record(app_deployment_crn)
|
||||
deployment_dir = os.path.join(deployment_parent_dir, fqdn)
|
||||
deployment_config_file = os.path.join(deployment_dir, "config.env")
|
||||
# TODO: Is there any reason not to simplify the hash input to the app_deployment_crn?
|
||||
deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest()
|
||||
app_image_shared_tag = f"laconic-webapp/{app.id}:local"
|
||||
# b. check for deployment directory (create if necessary)
|
||||
if not os.path.exists(deployment_dir):
|
||||
if deployment_record:
|
||||
@ -106,11 +110,20 @@ def process_app_deployment_request(
|
||||
needs_k8s_deploy = False
|
||||
# 6. build container (if needed)
|
||||
if not deployment_record or deployment_record.attributes.application != app.id:
|
||||
# TODO: pull from request
|
||||
extra_build_args = []
|
||||
needs_k8s_deploy = True
|
||||
# check if the image already exists
|
||||
shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag)
|
||||
if shared_tag_exists and not force_rebuild:
|
||||
# simply add our unique tag to the existing image and we are done
|
||||
print(f"Using existing app image {app_image_shared_tag} for {deployment_container_tag}", file=log_file)
|
||||
add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
|
||||
else:
|
||||
extra_build_args = [] # TODO: pull from request
|
||||
build_container_image(app, deployment_container_tag, extra_build_args, log_file)
|
||||
push_container_image(deployment_dir, log_file)
|
||||
needs_k8s_deploy = True
|
||||
# The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
|
||||
print(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_tag}", file=log_file)
|
||||
add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
|
||||
|
||||
# 7. update config (if needed)
|
||||
if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config:
|
||||
@ -171,12 +184,13 @@ def dump_known_requests(filename, requests, status="SEEN"):
|
||||
@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
|
||||
@click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="")
|
||||
@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="")
|
||||
@click.option("--force-rebuild", help="Rebuild even if the image already exists.", is_flag=True)
|
||||
@click.option("--log-dir", help="Output build/deployment logs to directory.", default=None)
|
||||
@click.pass_context
|
||||
def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, # noqa: C901
|
||||
request_id, discover, state_file, only_update_state,
|
||||
dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run,
|
||||
include_tags, exclude_tags, log_dir):
|
||||
include_tags, exclude_tags, force_rebuild, log_dir):
|
||||
if request_id and discover:
|
||||
print("Cannot specify both --request-id and --discover", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
@ -306,6 +320,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
|
||||
os.path.abspath(deployment_parent_dir),
|
||||
kube_config,
|
||||
image_registry,
|
||||
force_rebuild,
|
||||
run_log_file
|
||||
)
|
||||
status = "DEPLOYED"
|
||||
|
Loading…
Reference in New Issue
Block a user