From 1f4eb57069396a620dc9a1bcbc90a5441bc71833 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 13 Dec 2023 22:56:40 -0600 Subject: [PATCH] Add --dry-run option (#686) --- .../webapp/deploy_webapp_from_registry.py | 37 +- stack_orchestrator/deploy/webapp/util.py | 608 +++++++++--------- 2 files changed, 325 insertions(+), 320 deletions(-) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 5f9e712b..ad439812 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -164,10 +164,11 @@ def dump_known_requests(filename, requests): @click.option("--dns-suffix", help="DNS domain to use eg, laconic.servesthe.world") @click.option("--record-namespace-dns", help="eg, crn://laconic/dns") @click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments") +@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True) @click.pass_context def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, request_id, discover, state_file, only_update_state, - dns_suffix, record_namespace_dns, record_namespace_deployments): + dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run): if request_id and discover: print("Cannot specify both --request-id and --discover", file=sys.stderr) sys.exit(2) @@ -196,7 +197,8 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ requests = laconic.app_deployment_requests() if only_update_state: - dump_known_requests(state_file, requests) + if not dry_run: + dump_known_requests(state_file, requests) return previous_requests = load_known_requests(state_file) @@ -250,18 +252,19 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ print("Found %d unsatisfied request(s) to process." % len(requests_to_execute)) - for r in requests_to_execute: - try: - process_app_deployment_request( - ctx, - laconic, - r, - record_namespace_deployments, - record_namespace_dns, - dns_suffix, - deployment_parent_dir, - kube_config, - image_registry - ) - finally: - dump_known_requests(state_file, [r]) + if not dry_run: + for r in requests_to_execute: + try: + process_app_deployment_request( + ctx, + laconic, + r, + record_namespace_deployments, + record_namespace_dns, + dns_suffix, + deployment_parent_dir, + kube_config, + image_registry + ) + finally: + dump_known_requests(state_file, [r]) diff --git a/stack_orchestrator/deploy/webapp/util.py b/stack_orchestrator/deploy/webapp/util.py index ebc14c3f..f95d3f3e 100644 --- a/stack_orchestrator/deploy/webapp/util.py +++ b/stack_orchestrator/deploy/webapp/util.py @@ -1,303 +1,305 @@ -# Copyright © 2023 Vulcanize - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import hashlib -import json -import os -import random -import subprocess -import sys -import tempfile -import uuid - -import yaml - - -class AttrDict(dict): - def __init__(self, *args, **kwargs): - super(AttrDict, self).__init__(*args, **kwargs) - self.__dict__ = self - - def __getattribute__(self, attr): - __dict__ = super(AttrDict, self).__getattribute__("__dict__") - if attr in __dict__: - v = super(AttrDict, self).__getattribute__(attr) - if isinstance(v, dict): - return AttrDict(v) - return v - - -def cmd(*vargs): - try: - result = subprocess.run(vargs, capture_output=True) - result.check_returncode() - return result.stdout.decode() - except Exception as err: - print(result.stderr.decode()) - raise err - - -class LaconicRegistryClient: - def __init__(self, config_file): - self.config_file = config_file - self.cache = AttrDict( - { - "name_or_id": {}, - } - ) - - def list_records(self, criteria={}, all=False): - args = ["laconic", "-c", self.config_file, "cns", "record", "list"] - - if all: - args.append("--all") - - if criteria: - for k, v in criteria.items(): - args.append("--%s" % k) - args.append(str(v)) - - results = [AttrDict(r) for r in json.loads(cmd(*args))] - - # Most recent records first - results.sort(key=lambda r: r.createTime) - results.reverse() - - return results - - def is_crn(self, name_or_id: str): - if name_or_id: - return str(name_or_id).startswith("crn://") - return False - - def is_id(self, name_or_id: str): - return not self.is_crn(name_or_id) - - def _add_to_cache(self, records): - if not records: - return - - for p in records: - self.cache["name_or_id"][p.id] = p - if p.names: - for crn in p.names: - self.cache["name_or_id"][crn] = p - if p.attributes.type not in self.cache: - self.cache[p.attributes.type] = [] - self.cache[p.attributes.type].append(p) - - def resolve(self, name): - if not name: - return None - - if name in self.cache.name_or_id: - return self.cache.name_or_id[name] - - args = ["laconic", "-c", self.config_file, "cns", "name", "resolve", name] - - parsed = [AttrDict(r) for r in json.loads(cmd(*args))] - if parsed: - self._add_to_cache(parsed) - return parsed[0] - - return None - - def get_record(self, name_or_id, require=False): - if not name_or_id: - if require: - raise Exception("Cannot locate record:", name_or_id) - return None - - if name_or_id in self.cache.name_or_id: - return self.cache.name_or_id[name_or_id] - - if self.is_crn(name_or_id): - return self.resolve(name_or_id) - - args = [ - "laconic", - "-c", - self.config_file, - "cns", - "record", - "get", - "--id", - name_or_id, - ] - - parsed = [AttrDict(r) for r in json.loads(cmd(*args))] - if len(parsed): - self._add_to_cache(parsed) - return parsed[0] - - if require: - raise Exception("Cannot locate record:", name_or_id) - return None - - def app_deployment_requests(self): - return self.list_records({"type": "ApplicationDeploymentRequest"}, True) - - def app_deployments(self): - return self.list_records({"type": "ApplicationDeploymentRecord"}) - - def publish(self, record, names=[]): - tmpdir = tempfile.mkdtemp() - try: - record_fname = os.path.join(tmpdir, "record.yml") - record_file = open(record_fname, 'w') - yaml.dump(record, record_file) - record_file.close() - print(open(record_fname, 'r').read()) - - new_record_id = json.loads( - cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname) - )["id"] - for name in names: - cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id) - return new_record_id - finally: - cmd("rm", "-rf", tmpdir) - - -def file_hash(filename): - return hashlib.sha1(open(filename).read().encode()).hexdigest() - - -def build_container_image(app_record, tag, extra_build_args=[]): - tmpdir = tempfile.mkdtemp() - - try: - record_id = app_record["id"] - ref = app_record.attributes.repository_ref - repo = random.choice(app_record.attributes.repository) - clone_dir = os.path.join(tmpdir, record_id) - - print(f"Cloning repository {repo} to {clone_dir} ...") - if ref: - result = subprocess.run(["git", "clone", "--depth", "1", "--branch", ref, repo, clone_dir]) - result.check_returncode() - else: - result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir]) - result.check_returncode() - - print("Building webapp ...") - build_command = [sys.argv[0], "build-webapp", "--source-repo", clone_dir, "--tag", tag] - if extra_build_args: - build_command.append("--extra-build-args") - build_command.append(" ".join(extra_build_args)) - - result = subprocess.run(build_command) - result.check_returncode() - finally: - cmd("rm", "-rf", tmpdir) - - -def push_container_image(deployment_dir): - print("Pushing image ...") - result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"]) - result.check_returncode() - - -def deploy_to_k8s(deploy_record, deployment_dir): - if not deploy_record: - command = "up" - else: - command = "update" - - result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command]) - result.check_returncode() - - -def publish_deployment(laconic: LaconicRegistryClient, - app_record, - deploy_record, - deployment_crn, - dns_record, - dns_crn, - deployment_dir, - app_deployment_request=None): - if not deploy_record: - deploy_ver = "0.0.1" - else: - deploy_ver = "0.0.%d" % (int(deploy_record.attributes.version.split(".")[-1]) + 1) - - if not dns_record: - dns_ver = "0.0.1" - else: - dns_ver = "0.0.%d" % (int(dns_record.attributes.version.split(".")[-1]) + 1) - - spec = yaml.full_load(open(os.path.join(deployment_dir, "spec.yml"))) - fqdn = spec["network"]["http-proxy"][0]["host-name"] - - uniq = uuid.uuid4() - - new_dns_record = { - "record": { - "type": "DnsRecord", - "version": dns_ver, - "name": fqdn, - "resource_type": "A", - "meta": { - "so": uniq.hex - }, - } - } - if app_deployment_request: - new_dns_record["record"]["request"] = app_deployment_request.id - - dns_id = laconic.publish(new_dns_record, [dns_crn]) - - new_deployment_record = { - "record": { - "type": "ApplicationDeploymentRecord", - "version": deploy_ver, - "url": f"https://{fqdn}", - "name": app_record.attributes.name, - "application": app_record.id, - "dns": dns_id, - "meta": { - "config": file_hash(os.path.join(deployment_dir, "config.env")), - "so": uniq.hex - }, - } - } - if app_deployment_request: - new_deployment_record["record"]["request"] = app_deployment_request.id - - deployment_id = laconic.publish(new_deployment_record, [deployment_crn]) - return {"dns": dns_id, "deployment": deployment_id} - - -def hostname_for_deployment_request(app_deployment_request, laconic): - dns_name = app_deployment_request.attributes.dns - if not dns_name: - app = laconic.get_record(app_deployment_request.attributes.application, require=True) - dns_name = generate_hostname_for_app(app) - elif dns_name.startswith("crn://"): - record = laconic.get_record(dns_name, require=True) - dns_name = record.attributes.name - return dns_name - - -def generate_hostname_for_app(app): - last_part = app.attributes.name.split("/")[-1] - m = hashlib.sha256() - m.update(app.attributes.name.encode()) - m.update(b"|") - if isinstance(app.attributes.repository, list): - m.update(app.attributes.repository[0].encode()) - else: - m.update(app.attributes.repository.encode()) - return "%s-%s" % (last_part, m.hexdigest()[0:10]) +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import hashlib +import json +import os +import random +import subprocess +import sys +import tempfile +import uuid + +import yaml + + +class AttrDict(dict): + def __init__(self, *args, **kwargs): + super(AttrDict, self).__init__(*args, **kwargs) + self.__dict__ = self + + def __getattribute__(self, attr): + __dict__ = super(AttrDict, self).__getattribute__("__dict__") + if attr in __dict__: + v = super(AttrDict, self).__getattribute__(attr) + if isinstance(v, dict): + return AttrDict(v) + return v + + +def cmd(*vargs): + try: + result = subprocess.run(vargs, capture_output=True) + result.check_returncode() + return result.stdout.decode() + except Exception as err: + print(result.stderr.decode()) + raise err + + +class LaconicRegistryClient: + def __init__(self, config_file): + self.config_file = config_file + self.cache = AttrDict( + { + "name_or_id": {}, + } + ) + + def list_records(self, criteria={}, all=False): + args = ["laconic", "-c", self.config_file, "cns", "record", "list"] + + if all: + args.append("--all") + + if criteria: + for k, v in criteria.items(): + args.append("--%s" % k) + args.append(str(v)) + + results = [AttrDict(r) for r in json.loads(cmd(*args))] + + # Most recent records first + results.sort(key=lambda r: r.createTime) + results.reverse() + + return results + + def is_crn(self, name_or_id: str): + if name_or_id: + return str(name_or_id).startswith("crn://") + return False + + def is_id(self, name_or_id: str): + return not self.is_crn(name_or_id) + + def _add_to_cache(self, records): + if not records: + return + + for p in records: + self.cache["name_or_id"][p.id] = p + if p.names: + for crn in p.names: + self.cache["name_or_id"][crn] = p + if p.attributes.type not in self.cache: + self.cache[p.attributes.type] = [] + self.cache[p.attributes.type].append(p) + + def resolve(self, name): + if not name: + return None + + if name in self.cache.name_or_id: + return self.cache.name_or_id[name] + + args = ["laconic", "-c", self.config_file, "cns", "name", "resolve", name] + + parsed = [AttrDict(r) for r in json.loads(cmd(*args))] + if parsed: + self._add_to_cache(parsed) + return parsed[0] + + return None + + def get_record(self, name_or_id, require=False): + if not name_or_id: + if require: + raise Exception("Cannot locate record:", name_or_id) + return None + + if name_or_id in self.cache.name_or_id: + return self.cache.name_or_id[name_or_id] + + if self.is_crn(name_or_id): + return self.resolve(name_or_id) + + args = [ + "laconic", + "-c", + self.config_file, + "cns", + "record", + "get", + "--id", + name_or_id, + ] + + parsed = [AttrDict(r) for r in json.loads(cmd(*args))] + if len(parsed): + self._add_to_cache(parsed) + return parsed[0] + + if require: + raise Exception("Cannot locate record:", name_or_id) + return None + + def app_deployment_requests(self): + return self.list_records({"type": "ApplicationDeploymentRequest"}, True) + + def app_deployments(self): + return self.list_records({"type": "ApplicationDeploymentRecord"}) + + def publish(self, record, names=[]): + tmpdir = tempfile.mkdtemp() + try: + record_fname = os.path.join(tmpdir, "record.yml") + record_file = open(record_fname, 'w') + yaml.dump(record, record_file) + record_file.close() + print(open(record_fname, 'r').read()) + + new_record_id = json.loads( + cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname) + )["id"] + for name in names: + cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id) + return new_record_id + finally: + cmd("rm", "-rf", tmpdir) + + +def file_hash(filename): + return hashlib.sha1(open(filename).read().encode()).hexdigest() + + +def build_container_image(app_record, tag, extra_build_args=[]): + tmpdir = tempfile.mkdtemp() + + try: + record_id = app_record["id"] + ref = app_record.attributes.repository_ref + repo = random.choice(app_record.attributes.repository) + clone_dir = os.path.join(tmpdir, record_id) + + print(f"Cloning repository {repo} to {clone_dir} ...") + if ref: + # TODO: Determing branch or hash, and use depth 1 if we can. + result = subprocess.run(["git", "clone", repo, clone_dir]) + result.check_returncode() + subprocess.check_call(["git", "checkout", ref], cwd=clone_dir) + else: + result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir]) + result.check_returncode() + + print("Building webapp ...") + build_command = [sys.argv[0], "build-webapp", "--source-repo", clone_dir, "--tag", tag] + if extra_build_args: + build_command.append("--extra-build-args") + build_command.append(" ".join(extra_build_args)) + + result = subprocess.run(build_command) + result.check_returncode() + finally: + cmd("rm", "-rf", tmpdir) + + +def push_container_image(deployment_dir): + print("Pushing image ...") + result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"]) + result.check_returncode() + + +def deploy_to_k8s(deploy_record, deployment_dir): + if not deploy_record: + command = "up" + else: + command = "update" + + result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command]) + result.check_returncode() + + +def publish_deployment(laconic: LaconicRegistryClient, + app_record, + deploy_record, + deployment_crn, + dns_record, + dns_crn, + deployment_dir, + app_deployment_request=None): + if not deploy_record: + deploy_ver = "0.0.1" + else: + deploy_ver = "0.0.%d" % (int(deploy_record.attributes.version.split(".")[-1]) + 1) + + if not dns_record: + dns_ver = "0.0.1" + else: + dns_ver = "0.0.%d" % (int(dns_record.attributes.version.split(".")[-1]) + 1) + + spec = yaml.full_load(open(os.path.join(deployment_dir, "spec.yml"))) + fqdn = spec["network"]["http-proxy"][0]["host-name"] + + uniq = uuid.uuid4() + + new_dns_record = { + "record": { + "type": "DnsRecord", + "version": dns_ver, + "name": fqdn, + "resource_type": "A", + "meta": { + "so": uniq.hex + }, + } + } + if app_deployment_request: + new_dns_record["record"]["request"] = app_deployment_request.id + + dns_id = laconic.publish(new_dns_record, [dns_crn]) + + new_deployment_record = { + "record": { + "type": "ApplicationDeploymentRecord", + "version": deploy_ver, + "url": f"https://{fqdn}", + "name": app_record.attributes.name, + "application": app_record.id, + "dns": dns_id, + "meta": { + "config": file_hash(os.path.join(deployment_dir, "config.env")), + "so": uniq.hex + }, + } + } + if app_deployment_request: + new_deployment_record["record"]["request"] = app_deployment_request.id + + deployment_id = laconic.publish(new_deployment_record, [deployment_crn]) + return {"dns": dns_id, "deployment": deployment_id} + + +def hostname_for_deployment_request(app_deployment_request, laconic): + dns_name = app_deployment_request.attributes.dns + if not dns_name: + app = laconic.get_record(app_deployment_request.attributes.application, require=True) + dns_name = generate_hostname_for_app(app) + elif dns_name.startswith("crn://"): + record = laconic.get_record(dns_name, require=True) + dns_name = record.attributes.name + return dns_name + + +def generate_hostname_for_app(app): + last_part = app.attributes.name.split("/")[-1] + m = hashlib.sha256() + m.update(app.attributes.name.encode()) + m.update(b"|") + if isinstance(app.attributes.repository, list): + m.update(app.attributes.repository[0].encode()) + else: + m.update(app.attributes.repository.encode()) + return "%s-%s" % (last_part, m.hexdigest()[0:10])