Add the ability to filter deployment requests by tag. (#730)

Reviewed-on: cerc-io/stack-orchestrator#730
This commit is contained in:
Thomas E Lackey 2024-02-07 03:12:40 +00:00
parent 937b983ec9
commit 88a0236ca9
2 changed files with 68 additions and 26 deletions

View File

@ -19,6 +19,8 @@ import shlex
import shutil import shutil
import sys import sys
import tempfile import tempfile
import time
import uuid
import click import click
@ -39,8 +41,19 @@ def process_app_deployment_request(
dns_suffix, dns_suffix,
deployment_parent_dir, deployment_parent_dir,
kube_config, kube_config,
image_registry image_registry,
log_parent_dir
): ):
run_id = f"{app_deployment_request.id}-{str(time.time()).split('.')[0]}-{str(uuid.uuid4()).split('-')[0]}"
log_file = None
if log_parent_dir:
log_dir = os.path.join(log_parent_dir, app_deployment_request.id)
if not os.path.exists(log_dir):
os.mkdir(log_dir)
log_file_path = os.path.join(log_dir, f"{run_id}.log")
print(f"Directing build logs to: {log_file_path}")
log_file = open(log_file_path, "wt")
# 1. look up application # 1. look up application
app = laconic.get_record(app_deployment_request.attributes.application, require=True) app = laconic.get_record(app_deployment_request.attributes.application, require=True)
@ -102,8 +115,10 @@ def process_app_deployment_request(
needs_k8s_deploy = False needs_k8s_deploy = False
# 6. build container (if needed) # 6. build container (if needed)
if not deployment_record or deployment_record.attributes.application != app.id: if not deployment_record or deployment_record.attributes.application != app.id:
build_container_image(app, deployment_container_tag) # TODO: pull from request
push_container_image(deployment_dir) extra_build_args = []
build_container_image(app, deployment_container_tag, extra_build_args, log_file)
push_container_image(deployment_dir, log_file)
needs_k8s_deploy = True needs_k8s_deploy = True
# 7. update config (if needed) # 7. update config (if needed)
@ -116,6 +131,7 @@ def process_app_deployment_request(
deploy_to_k8s( deploy_to_k8s(
deployment_record, deployment_record,
deployment_dir, deployment_dir,
log_file
) )
publish_deployment( publish_deployment(
@ -162,10 +178,14 @@ def dump_known_requests(filename, requests, status="SEEN"):
@click.option("--record-namespace-dns", help="eg, crn://laconic/dns") @click.option("--record-namespace-dns", help="eg, crn://laconic/dns")
@click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments") @click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments")
@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True) @click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
@click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="")
@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="")
@click.option("--log-dir", help="Output build/deployment logs to directory.", default=None)
@click.pass_context @click.pass_context
def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, # noqa: C901
request_id, discover, state_file, only_update_state, request_id, discover, state_file, only_update_state,
dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run): dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run,
include_tags, exclude_tags, log_dir):
if request_id and discover: if request_id and discover:
print("Cannot specify both --request-id and --discover", file=sys.stderr) print("Cannot specify both --request-id and --discover", file=sys.stderr)
sys.exit(2) sys.exit(2)
@ -183,6 +203,10 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr) print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr)
sys.exit(2) sys.exit(2)
# Split CSV and clean up values.
include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
laconic = LaconicRegistryClient(laconic_config) laconic = LaconicRegistryClient(laconic_config)
# Find deployment requests. # Find deployment requests.
@ -198,12 +222,24 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
dump_known_requests(state_file, requests) dump_known_requests(state_file, requests)
return return
def skip_by_tag(r):
for tag in exclude_tags:
if tag and r.attributes.tags and tag in r.attributes.tags:
return True
for tag in include_tags:
if tag and (not r.attributes.tags or tag not in r.attributes.tags):
return True
return False
previous_requests = load_known_requests(state_file) previous_requests = load_known_requests(state_file)
# Collapse related requests. # Collapse related requests.
requests.sort(key=lambda r: r.createTime) requests.sort(key=lambda r: r.createTime)
requests.reverse() requests.reverse()
requests_by_name = {} requests_by_name = {}
skipped_by_name = {}
for r in requests: for r in requests:
# TODO: Do this _after_ filtering deployments and cancellations to minimize round trips. # TODO: Do this _after_ filtering deployments and cancellations to minimize round trips.
app = laconic.get_record(r.attributes.application) app = laconic.get_record(r.attributes.application)
@ -216,17 +252,20 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
requested_name = generate_hostname_for_app(app) requested_name = generate_hostname_for_app(app)
print("Generating name %s for request %s." % (requested_name, r.id)) print("Generating name %s for request %s." % (requested_name, r.id))
if requested_name not in requests_by_name: if requested_name in skipped_by_name or requested_name in requests_by_name:
print( print("Ignoring request %s, it has been superseded." % r.id)
"Found request %s to run application %s on %s." continue
% (r.id, r.attributes.application, requested_name)
) if skip_by_tag(r):
requests_by_name[requested_name] = r print("Skipping request %s, filtered by tag (include %s, exclude %s, present %s)" % (r.id,
else: include_tags,
print( exclude_tags,
"Ignoring request %s, it is superseded by %s." r.attributes.tags))
% (r.id, requests_by_name[requested_name].id) skipped_by_name[requested_name] = r
) continue
print("Found request %s to run application %s on %s." % (r.id, r.attributes.application, requested_name))
requests_by_name[requested_name] = r
# Find deployments. # Find deployments.
deployments = laconic.app_deployments() deployments = laconic.app_deployments()
@ -273,7 +312,8 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
dns_suffix, dns_suffix,
os.path.abspath(deployment_parent_dir), os.path.abspath(deployment_parent_dir),
kube_config, kube_config,
image_registry image_registry,
log_dir
) )
status = "DEPLOYED" status = "DEPLOYED"
finally: finally:

View File

@ -212,7 +212,7 @@ def determine_base_container(clone_dir, app_type="webapp"):
return base_container return base_container
def build_container_image(app_record, tag, extra_build_args=[]): def build_container_image(app_record, tag, extra_build_args=[], log_file=None):
tmpdir = tempfile.mkdtemp() tmpdir = tempfile.mkdtemp()
try: try:
@ -227,10 +227,10 @@ def build_container_image(app_record, tag, extra_build_args=[]):
git_env = dict(os.environ.copy()) git_env = dict(os.environ.copy())
# Never prompt # Never prompt
git_env["GIT_TERMINAL_PROMPT"] = "0" git_env["GIT_TERMINAL_PROMPT"] = "0"
subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env) subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env, stdout=log_file, stderr=log_file)
subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env) subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env, stdout=log_file, stderr=log_file)
else: else:
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir]) result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir], stdout=log_file, stderr=log_file)
result.check_returncode() result.check_returncode()
base_container = determine_base_container(clone_dir, app_record.attributes.app_type) base_container = determine_base_container(clone_dir, app_record.attributes.app_type)
@ -246,25 +246,27 @@ def build_container_image(app_record, tag, extra_build_args=[]):
build_command.append("--extra-build-args") build_command.append("--extra-build-args")
build_command.append(" ".join(extra_build_args)) build_command.append(" ".join(extra_build_args))
result = subprocess.run(build_command) result = subprocess.run(build_command, stdout=log_file, stderr=log_file)
result.check_returncode() result.check_returncode()
finally: finally:
cmd("rm", "-rf", tmpdir) cmd("rm", "-rf", tmpdir)
def push_container_image(deployment_dir): def push_container_image(deployment_dir, log_file=None):
print("Pushing image ...") print("Pushing image ...")
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"]) result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"],
stdout=log_file, stderr=log_file)
result.check_returncode() result.check_returncode()
def deploy_to_k8s(deploy_record, deployment_dir): def deploy_to_k8s(deploy_record, deployment_dir, log_file=None):
if not deploy_record: if not deploy_record:
command = "up" command = "up"
else: else:
command = "update" command = "update"
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command]) result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command],
stdout=log_file, stderr=log_file)
result.check_returncode() result.check_returncode()