forked from cerc-io/stack-orchestrator
Support application removal requests. (#697)
* Support application removal request. * Git should never prompt when deploying a webapp
This commit is contained in:
parent
a57b0cdd26
commit
837e443800
@ -26,7 +26,8 @@ from stack_orchestrator.deploy.webapp import deploy_webapp
|
||||
from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
|
||||
build_container_image, push_container_image,
|
||||
file_hash, deploy_to_k8s, publish_deployment,
|
||||
hostname_for_deployment_request, generate_hostname_for_app)
|
||||
hostname_for_deployment_request, generate_hostname_for_app,
|
||||
match_owner)
|
||||
|
||||
|
||||
def process_app_deployment_request(
|
||||
@ -57,19 +58,12 @@ def process_app_deployment_request(
|
||||
dns_crn = f"{dns_record_namespace}/{fqdn}"
|
||||
dns_record = laconic.get_record(dns_crn)
|
||||
if dns_record:
|
||||
dns_record_owners = dns_record.owners
|
||||
dns_request_owners = []
|
||||
if dns_record.request:
|
||||
prev_request = laconic.get_record(dns_record.request, require=True)
|
||||
dns_request_owners = prev_request.owners
|
||||
matched_owner = match_owner(app_deployment_request, dns_record)
|
||||
if not matched_owner and dns_record.request:
|
||||
matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.request, require=True))
|
||||
|
||||
owner_match = None
|
||||
|
||||
for owner in app_deployment_request.owners:
|
||||
if owner in dns_request_owners or owner in dns_record_owners:
|
||||
owner_match = owner
|
||||
if owner_match:
|
||||
print("Matched DnsRecord ownership to", owner)
|
||||
if matched_owner:
|
||||
print("Matched DnsRecord ownership:", matched_owner)
|
||||
else:
|
||||
raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" %
|
||||
(dns_record.id, app_deployment_request.id))
|
||||
@ -153,7 +147,6 @@ def dump_known_requests(filename, requests):
|
||||
|
||||
@click.command()
|
||||
@click.option("--kube-config", help="Provide a config file for a k8s deployment")
|
||||
@click.option("--kube-config", help="Provide a config file for a k8s deployment")
|
||||
@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
|
||||
@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster")
|
||||
@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True)
|
||||
@ -237,9 +230,20 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
|
||||
if d.attributes.request:
|
||||
deployments_by_request[d.attributes.request] = d
|
||||
|
||||
# Find removal requests.
|
||||
cancellation_requests = {}
|
||||
removal_requests = laconic.app_deployment_removal_requests()
|
||||
for r in removal_requests:
|
||||
if r.attributes.request:
|
||||
cancellation_requests[r.attributes.request] = r
|
||||
|
||||
requests_to_execute = []
|
||||
for r in requests_by_name.values():
|
||||
if r.id not in deployments_by_request:
|
||||
if r.id in cancellation_requests and match_owner(cancellation_requests[r.id], r):
|
||||
print(f"Found deployment cancellation request for {r.id} at {cancellation_requests[r.id].id}")
|
||||
elif r.id in deployments_by_request:
|
||||
print(f"Found satisfied request for {r.id} at {deployments_by_request[r.id].id}")
|
||||
else:
|
||||
if r.id not in previous_requests:
|
||||
print(f"Request {r.id} needs to processed.")
|
||||
requests_to_execute.append(r)
|
||||
@ -247,8 +251,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
|
||||
print(
|
||||
f"Skipping unsatisfied request {r.id} because we have seen it before."
|
||||
)
|
||||
else:
|
||||
print(f"Found satisfied request {r.id} at {deployments_by_request[r.id].names[0]}")
|
||||
|
||||
print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))
|
||||
|
||||
|
@ -0,0 +1,189 @@
|
||||
# Copyright ©2023 Vulcanize
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import click
|
||||
|
||||
from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient, match_owner
|
||||
|
||||
|
||||
def process_app_removal_request(ctx,
|
||||
laconic: LaconicRegistryClient,
|
||||
app_removal_request,
|
||||
deployment_parent_dir,
|
||||
delete_volumes,
|
||||
delete_names):
|
||||
deployment_record = laconic.get_record(app_removal_request.attributes.deployment, require=True)
|
||||
dns_record = laconic.get_record(deployment_record.attributes.dns, require=True)
|
||||
deployment_dir = os.path.join(deployment_parent_dir, dns_record.attributes.name)
|
||||
|
||||
if not os.path.exists(deployment_dir):
|
||||
raise Exception("Deployment directory %s does not exist." % deployment_dir)
|
||||
|
||||
# Check if the removal request is from the owner of the DnsRecord or deployment record.
|
||||
matched_owner = match_owner(app_removal_request, deployment_record, dns_record)
|
||||
|
||||
# Or of the original deployment request.
|
||||
if not matched_owner and deployment_record.request:
|
||||
matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.request, require=True))
|
||||
|
||||
if matched_owner:
|
||||
print("Matched deployment ownership:", matched_owner)
|
||||
else:
|
||||
raise Exception("Unable to confirm ownership of deployment %s for removal request %s" %
|
||||
(deployment_record.id, app_removal_request.id))
|
||||
|
||||
# TODO(telackey): Call the function directly. The easiest way to build the correct click context is to
|
||||
# exec the process, but it would be better to refactor so we could just call down_operation with the
|
||||
# necessary parameters
|
||||
down_command = [sys.argv[0], "deployment", "--dir", deployment_dir, "down"]
|
||||
if delete_volumes:
|
||||
down_command.append("--delete-volumes")
|
||||
result = subprocess.run(down_command)
|
||||
result.check_returncode()
|
||||
|
||||
removal_record = {
|
||||
"record": {
|
||||
"type": "ApplicationDeploymentRemovalRecord",
|
||||
"version": "1.0.0",
|
||||
"request": app_removal_request.id,
|
||||
"deployment": deployment_record.id,
|
||||
}
|
||||
}
|
||||
laconic.publish(removal_record)
|
||||
|
||||
if delete_names:
|
||||
if deployment_record.names:
|
||||
for name in deployment_record.names:
|
||||
laconic.delete_name(name)
|
||||
|
||||
if dns_record.names:
|
||||
for name in dns_record.names:
|
||||
laconic.delete_name(name)
|
||||
|
||||
if delete_volumes:
|
||||
shutil.rmtree(deployment_dir)
|
||||
|
||||
|
||||
def load_known_requests(filename):
|
||||
if filename and os.path.exists(filename):
|
||||
return json.load(open(filename, "r"))
|
||||
return {}
|
||||
|
||||
|
||||
def dump_known_requests(filename, requests):
|
||||
if not filename:
|
||||
return
|
||||
known_requests = load_known_requests(filename)
|
||||
for r in requests:
|
||||
known_requests[r.id] = r.createTime
|
||||
json.dump(known_requests, open(filename, "w"))
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
|
||||
@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True)
|
||||
@click.option("--request-id", help="The ApplicationDeploymentRemovalRequest to process")
|
||||
@click.option("--discover", help="Discover and process all pending ApplicationDeploymentRemovalRequests",
|
||||
is_flag=True, default=False)
|
||||
@click.option("--state-file", help="File to store state about previously seen requests.")
|
||||
@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True)
|
||||
@click.option("--delete-names/--preserve-names", help="Delete all names associated with removed deployments.", default=True)
|
||||
@click.option("--delete-volumes/--preserve-volumes", default=True, help="delete data volumes")
|
||||
@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
|
||||
@click.pass_context
|
||||
def command(ctx, laconic_config, deployment_parent_dir,
|
||||
request_id, discover, state_file, only_update_state,
|
||||
delete_names, delete_volumes, dry_run):
|
||||
if request_id and discover:
|
||||
print("Cannot specify both --request-id and --discover", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
if not request_id and not discover:
|
||||
print("Must specify either --request-id or --discover", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
if only_update_state and not state_file:
|
||||
print("--only-update-state requires --state-file", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
laconic = LaconicRegistryClient(laconic_config)
|
||||
|
||||
# Find deployment removal requests.
|
||||
# single request
|
||||
if request_id:
|
||||
requests = [laconic.get_record(request_id, require=True)]
|
||||
# TODO: assert record type
|
||||
# all requests
|
||||
elif discover:
|
||||
requests = laconic.app_deployment_removal_requests()
|
||||
|
||||
if only_update_state:
|
||||
if not dry_run:
|
||||
dump_known_requests(state_file, requests)
|
||||
return
|
||||
|
||||
previous_requests = load_known_requests(state_file)
|
||||
requests.sort(key=lambda r: r.createTime)
|
||||
|
||||
# Find deployments.
|
||||
deployments = {}
|
||||
for d in laconic.app_deployments(all=True):
|
||||
deployments[d.id] = d
|
||||
|
||||
# Find removal requests.
|
||||
removals_by_deployment = {}
|
||||
removals_by_request = {}
|
||||
for r in laconic.app_deployment_removals():
|
||||
if r.attributes.deployment:
|
||||
# TODO: should we handle CRNs?
|
||||
removals_by_deployment[r.attributes.deployment] = r
|
||||
|
||||
requests_to_execute = []
|
||||
for r in requests:
|
||||
if not r.attributes.deployment:
|
||||
print(f"Skipping removal request {r.id} since it was a cancellation.")
|
||||
elif r.id in removals_by_request:
|
||||
print(f"Found satisfied request for {r.id} at {removals_by_request[r.id].id}")
|
||||
elif r.attributes.deployment in removals_by_deployment:
|
||||
print(
|
||||
f"Found removal record for indicated deployment {r.attributes.deployment} at "
|
||||
f"{removals_by_deployment[r.attributes.deployment].id}")
|
||||
else:
|
||||
if r.id not in previous_requests:
|
||||
print(f"Request {r.id} needs to processed.")
|
||||
requests_to_execute.append(r)
|
||||
else:
|
||||
print(f"Skipping unsatisfied request {r.id} because we have seen it before.")
|
||||
|
||||
print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))
|
||||
|
||||
if not dry_run:
|
||||
for r in requests_to_execute:
|
||||
try:
|
||||
process_app_removal_request(
|
||||
ctx,
|
||||
laconic,
|
||||
r,
|
||||
os.path.abspath(deployment_parent_dir),
|
||||
delete_volumes,
|
||||
delete_names
|
||||
)
|
||||
finally:
|
||||
dump_known_requests(state_file, [r])
|
@ -49,6 +49,14 @@ def cmd(*vargs):
|
||||
raise err
|
||||
|
||||
|
||||
def match_owner(recordA, *records):
|
||||
for owner in recordA.owners:
|
||||
for otherRecord in records:
|
||||
if owner in otherRecord.owners:
|
||||
return owner
|
||||
return None
|
||||
|
||||
|
||||
class LaconicRegistryClient:
|
||||
def __init__(self, config_file):
|
||||
self.config_file = config_file
|
||||
@ -146,11 +154,17 @@ class LaconicRegistryClient:
|
||||
raise Exception("Cannot locate record:", name_or_id)
|
||||
return None
|
||||
|
||||
def app_deployment_requests(self):
|
||||
return self.list_records({"type": "ApplicationDeploymentRequest"}, True)
|
||||
def app_deployment_requests(self, all=True):
|
||||
return self.list_records({"type": "ApplicationDeploymentRequest"}, all)
|
||||
|
||||
def app_deployments(self):
|
||||
return self.list_records({"type": "ApplicationDeploymentRecord"})
|
||||
def app_deployments(self, all=True):
|
||||
return self.list_records({"type": "ApplicationDeploymentRecord"}, all)
|
||||
|
||||
def app_deployment_removal_requests(self, all=True):
|
||||
return self.list_records({"type": "ApplicationDeploymentRemovalRequest"}, all)
|
||||
|
||||
def app_deployment_removals(self, all=True):
|
||||
return self.list_records({"type": "ApplicationDeploymentRemovalRecord"}, all)
|
||||
|
||||
def publish(self, record, names=[]):
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
@ -165,11 +179,17 @@ class LaconicRegistryClient:
|
||||
cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname)
|
||||
)["id"]
|
||||
for name in names:
|
||||
cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id)
|
||||
self.set_name(name, new_record_id)
|
||||
return new_record_id
|
||||
finally:
|
||||
cmd("rm", "-rf", tmpdir)
|
||||
|
||||
def set_name(self, name, record_id):
|
||||
cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, record_id)
|
||||
|
||||
def delete_name(self, name):
|
||||
cmd("laconic", "-c", self.config_file, "cns", "name", "delete", name)
|
||||
|
||||
|
||||
def file_hash(filename):
|
||||
return hashlib.sha1(open(filename).read().encode()).hexdigest()
|
||||
@ -187,9 +207,11 @@ def build_container_image(app_record, tag, extra_build_args=[]):
|
||||
print(f"Cloning repository {repo} to {clone_dir} ...")
|
||||
if ref:
|
||||
# TODO: Determing branch or hash, and use depth 1 if we can.
|
||||
result = subprocess.run(["git", "clone", repo, clone_dir])
|
||||
result.check_returncode()
|
||||
subprocess.check_call(["git", "checkout", ref], cwd=clone_dir)
|
||||
git_env = dict(os.environ.copy())
|
||||
# Never prompt
|
||||
git_env["GIT_TERMINAL_PROMPT"] = "0"
|
||||
subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env)
|
||||
subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env)
|
||||
else:
|
||||
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir])
|
||||
result.check_returncode()
|
||||
|
@ -20,7 +20,10 @@ from stack_orchestrator.repos import setup_repositories
|
||||
from stack_orchestrator.build import build_containers
|
||||
from stack_orchestrator.build import build_npms
|
||||
from stack_orchestrator.build import build_webapp
|
||||
from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp, deploy_webapp_from_registry
|
||||
from stack_orchestrator.deploy.webapp import (run_webapp,
|
||||
deploy_webapp,
|
||||
deploy_webapp_from_registry,
|
||||
undeploy_webapp_from_registry)
|
||||
from stack_orchestrator.deploy import deploy
|
||||
from stack_orchestrator import version
|
||||
from stack_orchestrator.deploy import deployment
|
||||
@ -54,6 +57,7 @@ cli.add_command(build_webapp.command, "build-webapp")
|
||||
cli.add_command(run_webapp.command, "run-webapp")
|
||||
cli.add_command(deploy_webapp.command, "deploy-webapp")
|
||||
cli.add_command(deploy_webapp_from_registry.command, "deploy-webapp-from-registry")
|
||||
cli.add_command(undeploy_webapp_from_registry.command, "undeploy-webapp-from-registry")
|
||||
cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system
|
||||
cli.add_command(deploy.command, "deploy-system")
|
||||
cli.add_command(deployment.command, "deployment")
|
||||
|
Loading…
Reference in New Issue
Block a user