From 837e4438006bb142e8ec9be47e2a12bd8bc9fc9a Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 21 Dec 2023 18:05:40 -0600 Subject: [PATCH] Support application removal requests. (#697) * Support application removal request. * Git should never prompt when deploying a webapp --- .../webapp/deploy_webapp_from_registry.py | 36 ++-- .../webapp/undeploy_webapp_from_registry.py | 189 ++++++++++++++++++ stack_orchestrator/deploy/webapp/util.py | 38 +++- stack_orchestrator/main.py | 6 +- 4 files changed, 243 insertions(+), 26 deletions(-) create mode 100644 stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 02731992..2e4544eb 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -26,7 +26,8 @@ from stack_orchestrator.deploy.webapp import deploy_webapp from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, build_container_image, push_container_image, file_hash, deploy_to_k8s, publish_deployment, - hostname_for_deployment_request, generate_hostname_for_app) + hostname_for_deployment_request, generate_hostname_for_app, + match_owner) def process_app_deployment_request( @@ -57,19 +58,12 @@ def process_app_deployment_request( dns_crn = f"{dns_record_namespace}/{fqdn}" dns_record = laconic.get_record(dns_crn) if dns_record: - dns_record_owners = dns_record.owners - dns_request_owners = [] - if dns_record.request: - prev_request = laconic.get_record(dns_record.request, require=True) - dns_request_owners = prev_request.owners + matched_owner = match_owner(app_deployment_request, dns_record) + if not matched_owner and dns_record.request: + matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.request, require=True)) - owner_match = None - - for owner in app_deployment_request.owners: - if owner in dns_request_owners or owner in dns_record_owners: - owner_match = owner - if owner_match: - print("Matched DnsRecord ownership to", owner) + if matched_owner: + print("Matched DnsRecord ownership:", matched_owner) else: raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" % (dns_record.id, app_deployment_request.id)) @@ -153,7 +147,6 @@ def dump_known_requests(filename, requests): @click.command() @click.option("--kube-config", help="Provide a config file for a k8s deployment") -@click.option("--kube-config", help="Provide a config file for a k8s deployment") @click.option("--laconic-config", help="Provide a config file for laconicd", required=True) @click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") @click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True) @@ -237,9 +230,20 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ if d.attributes.request: deployments_by_request[d.attributes.request] = d + # Find removal requests. + cancellation_requests = {} + removal_requests = laconic.app_deployment_removal_requests() + for r in removal_requests: + if r.attributes.request: + cancellation_requests[r.attributes.request] = r + requests_to_execute = [] for r in requests_by_name.values(): - if r.id not in deployments_by_request: + if r.id in cancellation_requests and match_owner(cancellation_requests[r.id], r): + print(f"Found deployment cancellation request for {r.id} at {cancellation_requests[r.id].id}") + elif r.id in deployments_by_request: + print(f"Found satisfied request for {r.id} at {deployments_by_request[r.id].id}") + else: if r.id not in previous_requests: print(f"Request {r.id} needs to processed.") requests_to_execute.append(r) @@ -247,8 +251,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ print( f"Skipping unsatisfied request {r.id} because we have seen it before." ) - else: - print(f"Found satisfied request {r.id} at {deployments_by_request[r.id].names[0]}") print("Found %d unsatisfied request(s) to process." % len(requests_to_execute)) diff --git a/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py new file mode 100644 index 00000000..6c5cac85 --- /dev/null +++ b/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py @@ -0,0 +1,189 @@ +# Copyright ©2023 Vulcanize +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import json +import os +import shutil +import subprocess +import sys + +import click + +from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient, match_owner + + +def process_app_removal_request(ctx, + laconic: LaconicRegistryClient, + app_removal_request, + deployment_parent_dir, + delete_volumes, + delete_names): + deployment_record = laconic.get_record(app_removal_request.attributes.deployment, require=True) + dns_record = laconic.get_record(deployment_record.attributes.dns, require=True) + deployment_dir = os.path.join(deployment_parent_dir, dns_record.attributes.name) + + if not os.path.exists(deployment_dir): + raise Exception("Deployment directory %s does not exist." % deployment_dir) + + # Check if the removal request is from the owner of the DnsRecord or deployment record. + matched_owner = match_owner(app_removal_request, deployment_record, dns_record) + + # Or of the original deployment request. + if not matched_owner and deployment_record.request: + matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.request, require=True)) + + if matched_owner: + print("Matched deployment ownership:", matched_owner) + else: + raise Exception("Unable to confirm ownership of deployment %s for removal request %s" % + (deployment_record.id, app_removal_request.id)) + + # TODO(telackey): Call the function directly. The easiest way to build the correct click context is to + # exec the process, but it would be better to refactor so we could just call down_operation with the + # necessary parameters + down_command = [sys.argv[0], "deployment", "--dir", deployment_dir, "down"] + if delete_volumes: + down_command.append("--delete-volumes") + result = subprocess.run(down_command) + result.check_returncode() + + removal_record = { + "record": { + "type": "ApplicationDeploymentRemovalRecord", + "version": "1.0.0", + "request": app_removal_request.id, + "deployment": deployment_record.id, + } + } + laconic.publish(removal_record) + + if delete_names: + if deployment_record.names: + for name in deployment_record.names: + laconic.delete_name(name) + + if dns_record.names: + for name in dns_record.names: + laconic.delete_name(name) + + if delete_volumes: + shutil.rmtree(deployment_dir) + + +def load_known_requests(filename): + if filename and os.path.exists(filename): + return json.load(open(filename, "r")) + return {} + + +def dump_known_requests(filename, requests): + if not filename: + return + known_requests = load_known_requests(filename) + for r in requests: + known_requests[r.id] = r.createTime + json.dump(known_requests, open(filename, "w")) + + +@click.command() +@click.option("--laconic-config", help="Provide a config file for laconicd", required=True) +@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True) +@click.option("--request-id", help="The ApplicationDeploymentRemovalRequest to process") +@click.option("--discover", help="Discover and process all pending ApplicationDeploymentRemovalRequests", + is_flag=True, default=False) +@click.option("--state-file", help="File to store state about previously seen requests.") +@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True) +@click.option("--delete-names/--preserve-names", help="Delete all names associated with removed deployments.", default=True) +@click.option("--delete-volumes/--preserve-volumes", default=True, help="delete data volumes") +@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True) +@click.pass_context +def command(ctx, laconic_config, deployment_parent_dir, + request_id, discover, state_file, only_update_state, + delete_names, delete_volumes, dry_run): + if request_id and discover: + print("Cannot specify both --request-id and --discover", file=sys.stderr) + sys.exit(2) + + if not request_id and not discover: + print("Must specify either --request-id or --discover", file=sys.stderr) + sys.exit(2) + + if only_update_state and not state_file: + print("--only-update-state requires --state-file", file=sys.stderr) + sys.exit(2) + + laconic = LaconicRegistryClient(laconic_config) + + # Find deployment removal requests. + # single request + if request_id: + requests = [laconic.get_record(request_id, require=True)] + # TODO: assert record type + # all requests + elif discover: + requests = laconic.app_deployment_removal_requests() + + if only_update_state: + if not dry_run: + dump_known_requests(state_file, requests) + return + + previous_requests = load_known_requests(state_file) + requests.sort(key=lambda r: r.createTime) + + # Find deployments. + deployments = {} + for d in laconic.app_deployments(all=True): + deployments[d.id] = d + + # Find removal requests. + removals_by_deployment = {} + removals_by_request = {} + for r in laconic.app_deployment_removals(): + if r.attributes.deployment: + # TODO: should we handle CRNs? + removals_by_deployment[r.attributes.deployment] = r + + requests_to_execute = [] + for r in requests: + if not r.attributes.deployment: + print(f"Skipping removal request {r.id} since it was a cancellation.") + elif r.id in removals_by_request: + print(f"Found satisfied request for {r.id} at {removals_by_request[r.id].id}") + elif r.attributes.deployment in removals_by_deployment: + print( + f"Found removal record for indicated deployment {r.attributes.deployment} at " + f"{removals_by_deployment[r.attributes.deployment].id}") + else: + if r.id not in previous_requests: + print(f"Request {r.id} needs to processed.") + requests_to_execute.append(r) + else: + print(f"Skipping unsatisfied request {r.id} because we have seen it before.") + + print("Found %d unsatisfied request(s) to process." % len(requests_to_execute)) + + if not dry_run: + for r in requests_to_execute: + try: + process_app_removal_request( + ctx, + laconic, + r, + os.path.abspath(deployment_parent_dir), + delete_volumes, + delete_names + ) + finally: + dump_known_requests(state_file, [r]) diff --git a/stack_orchestrator/deploy/webapp/util.py b/stack_orchestrator/deploy/webapp/util.py index f95d3f3e..ef98117b 100644 --- a/stack_orchestrator/deploy/webapp/util.py +++ b/stack_orchestrator/deploy/webapp/util.py @@ -49,6 +49,14 @@ def cmd(*vargs): raise err +def match_owner(recordA, *records): + for owner in recordA.owners: + for otherRecord in records: + if owner in otherRecord.owners: + return owner + return None + + class LaconicRegistryClient: def __init__(self, config_file): self.config_file = config_file @@ -146,11 +154,17 @@ class LaconicRegistryClient: raise Exception("Cannot locate record:", name_or_id) return None - def app_deployment_requests(self): - return self.list_records({"type": "ApplicationDeploymentRequest"}, True) + def app_deployment_requests(self, all=True): + return self.list_records({"type": "ApplicationDeploymentRequest"}, all) - def app_deployments(self): - return self.list_records({"type": "ApplicationDeploymentRecord"}) + def app_deployments(self, all=True): + return self.list_records({"type": "ApplicationDeploymentRecord"}, all) + + def app_deployment_removal_requests(self, all=True): + return self.list_records({"type": "ApplicationDeploymentRemovalRequest"}, all) + + def app_deployment_removals(self, all=True): + return self.list_records({"type": "ApplicationDeploymentRemovalRecord"}, all) def publish(self, record, names=[]): tmpdir = tempfile.mkdtemp() @@ -165,11 +179,17 @@ class LaconicRegistryClient: cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname) )["id"] for name in names: - cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id) + self.set_name(name, new_record_id) return new_record_id finally: cmd("rm", "-rf", tmpdir) + def set_name(self, name, record_id): + cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, record_id) + + def delete_name(self, name): + cmd("laconic", "-c", self.config_file, "cns", "name", "delete", name) + def file_hash(filename): return hashlib.sha1(open(filename).read().encode()).hexdigest() @@ -187,9 +207,11 @@ def build_container_image(app_record, tag, extra_build_args=[]): print(f"Cloning repository {repo} to {clone_dir} ...") if ref: # TODO: Determing branch or hash, and use depth 1 if we can. - result = subprocess.run(["git", "clone", repo, clone_dir]) - result.check_returncode() - subprocess.check_call(["git", "checkout", ref], cwd=clone_dir) + git_env = dict(os.environ.copy()) + # Never prompt + git_env["GIT_TERMINAL_PROMPT"] = "0" + subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env) + subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env) else: result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir]) result.check_returncode() diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py index fabc58ff..a4644412 100644 --- a/stack_orchestrator/main.py +++ b/stack_orchestrator/main.py @@ -20,7 +20,10 @@ from stack_orchestrator.repos import setup_repositories from stack_orchestrator.build import build_containers from stack_orchestrator.build import build_npms from stack_orchestrator.build import build_webapp -from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp, deploy_webapp_from_registry +from stack_orchestrator.deploy.webapp import (run_webapp, + deploy_webapp, + deploy_webapp_from_registry, + undeploy_webapp_from_registry) from stack_orchestrator.deploy import deploy from stack_orchestrator import version from stack_orchestrator.deploy import deployment @@ -54,6 +57,7 @@ cli.add_command(build_webapp.command, "build-webapp") cli.add_command(run_webapp.command, "run-webapp") cli.add_command(deploy_webapp.command, "deploy-webapp") cli.add_command(deploy_webapp_from_registry.command, "deploy-webapp-from-registry") +cli.add_command(undeploy_webapp_from_registry.command, "undeploy-webapp-from-registry") cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system cli.add_command(deploy.command, "deploy-system") cli.add_command(deployment.command, "deployment")