From 88f66a36263a238b0993d7bbd5aac1244c7673c5 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 13 Dec 2023 21:02:34 -0600 Subject: [PATCH] Add `deployment update` and `deploy-webapp-from-registry` commands. (#676) --- .../demo-records/demo-record-10.yml | 14 + .../demo-records/demo-record-11.yml | 12 + .../demo-records/demo-record-12.yml | 17 + .../demo-records/demo-record-7.yml | 11 + .../demo-records/demo-record-8.yml | 18 ++ .../demo-records/demo-record-9.yml | 17 + .../deploy/compose/deploy_docker.py | 6 + stack_orchestrator/deploy/deploy.py | 8 + stack_orchestrator/deploy/deployer.py | 4 + stack_orchestrator/deploy/deployment.py | 9 +- stack_orchestrator/deploy/k8s/cluster_info.py | 1 + stack_orchestrator/deploy/k8s/deploy_k8s.py | 29 ++ .../deploy/webapp/deploy_webapp.py | 47 +-- .../webapp/deploy_webapp_from_registry.py | 267 +++++++++++++++ stack_orchestrator/deploy/webapp/util.py | 303 ++++++++++++++++++ stack_orchestrator/main.py | 3 +- 16 files changed, 743 insertions(+), 23 deletions(-) create mode 100644 stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml create mode 100644 stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml create mode 100644 stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-12.yml create mode 100644 stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-7.yml create mode 100644 stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-8.yml create mode 100644 stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-9.yml create mode 100644 stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py create mode 100644 stack_orchestrator/deploy/webapp/util.py diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml new file mode 100644 index 00000000..a467903e --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml @@ -0,0 +1,14 @@ +record: + type: ApplicationDeploymentRecord + version: 1.2.3 + name: name + description: description + application: application + url: url + dns: dns + request: request + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml new file mode 100644 index 00000000..3afbd64d --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml @@ -0,0 +1,12 @@ +record: + type: GeneralRecord + version: 1.2.3 + name: name + description: description + category: category + value: value + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-12.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-12.yml new file mode 100644 index 00000000..4c5f024c --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-12.yml @@ -0,0 +1,17 @@ +record: + type: ApplicationArtifact + version: 1.2.3 + name: name + description: description + application: appidgoeshere + content_type: content_type + os: os + cpu: cpu + uri: + - uri://a + - uri://b + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-7.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-7.yml new file mode 100644 index 00000000..1bf4ad46 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-7.yml @@ -0,0 +1,11 @@ +record: + type: DnsRecord + version: 0.0.1 + name: "foo" + resource_type: "A" + value: "bar" + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-8.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-8.yml new file mode 100644 index 00000000..95bd195a --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-8.yml @@ -0,0 +1,18 @@ +record: + type: ApplicationRecord + version: 0.0.1 + name: my-demo-app + description: "Description of my app" + homepage: http://my.demo.app + license: license + author: author + repository: + - "https://my.demo.repo" + repository_ref: "v0.1.0" + app_version: "0.1.0" + app_type: "webapp" + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-9.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-9.yml new file mode 100644 index 00000000..415e5c74 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-9.yml @@ -0,0 +1,17 @@ +record: + type: ApplicationDeploymentRequest + version: 1.2.3 + application: application + dns: dns + config: + env: + ENV_VAR_A: A + ENV_VAR_B: B + crn: + - crn://foo.bar + - crn://bar.baz + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/deploy/compose/deploy_docker.py b/stack_orchestrator/deploy/compose/deploy_docker.py index d34d1e6f..b2622820 100644 --- a/stack_orchestrator/deploy/compose/deploy_docker.py +++ b/stack_orchestrator/deploy/compose/deploy_docker.py @@ -40,6 +40,12 @@ class DockerDeployer(Deployer): except DockerException as e: raise DeployerException(e) + def update(self): + try: + return self.docker.compose.restart() + except DockerException as e: + raise DeployerException(e) + def status(self): try: for p in self.docker.compose.ps(): diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index da96a500..18d27a21 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -120,6 +120,14 @@ def status_operation(ctx): ctx.obj.deployer.status() +def update_operation(ctx): + global_context = ctx.parent.parent.obj + if not global_context.dry_run: + if global_context.verbose: + print("Running compose update") + ctx.obj.deployer.update() + + def ps_operation(ctx): global_context = ctx.parent.parent.obj if not global_context.dry_run: diff --git a/stack_orchestrator/deploy/deployer.py b/stack_orchestrator/deploy/deployer.py index 2806044b..2df784a2 100644 --- a/stack_orchestrator/deploy/deployer.py +++ b/stack_orchestrator/deploy/deployer.py @@ -27,6 +27,10 @@ class Deployer(ABC): def down(self, timeout, volumes): pass + @abstractmethod + def update(self): + pass + @abstractmethod def ps(self): pass diff --git a/stack_orchestrator/deploy/deployment.py b/stack_orchestrator/deploy/deployment.py index 366a83f6..f364121f 100644 --- a/stack_orchestrator/deploy/deployment.py +++ b/stack_orchestrator/deploy/deployment.py @@ -19,7 +19,7 @@ import sys from stack_orchestrator import constants from stack_orchestrator.deploy.images import push_images_operation from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation, status_operation -from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context +from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context, update_operation from stack_orchestrator.deploy.deploy_types import DeployCommandContext from stack_orchestrator.deploy.deployment_context import DeploymentContext @@ -149,3 +149,10 @@ def logs(ctx, tail, follow, extra_args): def status(ctx): ctx.obj = make_deploy_context(ctx) status_operation(ctx) + + +@command.command() +@click.pass_context +def update(ctx): + ctx.obj = make_deploy_context(ctx) + update_operation(ctx) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 0aa74189..24fe15a0 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -189,6 +189,7 @@ class ClusterInfo: container = client.V1Container( name=container_name, image=image_to_use, + image_pull_policy="Always", env=envs_from_environment_variables_map(self.environment_variables.map), ports=[client.V1ContainerPort(container_port=port)], volume_mounts=volume_mounts, diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 95131966..bf82ebdf 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -13,6 +13,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from datetime import datetime, timezone + from pathlib import Path from kubernetes import client, config @@ -304,6 +306,33 @@ class K8sDeployer(Deployer): log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test") return log_stream_from_string(log_data) + def update(self): + self.connect_api() + ref_deployment = self.cluster_info.get_deployment() + + deployment = self.apps_api.read_namespaced_deployment( + name=ref_deployment.metadata.name, + namespace=self.k8s_namespace + ) + + new_env = ref_deployment.spec.template.spec.containers[0].env + for container in deployment.spec.template.spec.containers: + old_env = container.env + if old_env != new_env: + container.env = new_env + + deployment.spec.template.metadata.annotations = { + "kubectl.kubernetes.io/restartedAt": datetime.utcnow() + .replace(tzinfo=timezone.utc) + .isoformat() + } + + self.apps_api.patch_namespaced_deployment( + name=ref_deployment.metadata.name, + namespace=self.k8s_namespace, + body=deployment + ) + def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False): # We need to figure out how to do this -- check why we're being called first pass diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp.py b/stack_orchestrator/deploy/webapp/deploy_webapp.py index 391162c9..aca2df35 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp.py @@ -54,26 +54,7 @@ def _fixup_url_spec(spec_file_name: str, url: str): wfile.write(contents) -@click.group() -@click.pass_context -def command(ctx): - '''manage a webapp deployment''' - - # Check that --stack wasn't supplied - if ctx.parent.obj.stack: - error_exit("--stack can't be supplied with the deploy-webapp command") - - -@command.command() -@click.option("--kube-config", help="Provide a config file for a k8s deployment") -@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") -@click.option("--deployment-dir", help="Create deployment files in this directory", required=True) -@click.option("--image", help="image to deploy", required=True) -@click.option("--url", help="url to serve", required=True) -@click.option("--env-file", help="environment file for webapp") -@click.pass_context -def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file): - '''create a deployment for the specified webapp container''' +def create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file): # Do the equivalent of: # 1. laconic-so --stack webapp-template deploy --deploy-to k8s init --output webapp-spec.yml # --config (eqivalent of the contents of my-config.env) @@ -92,7 +73,7 @@ def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_fil # TODO: support env file deploy_command_context: DeployCommandContext = create_deploy_context( global_options2(ctx), None, stack, None, None, None, env_file, "k8s" - ) + ) init_operation( deploy_command_context, stack, @@ -116,3 +97,27 @@ def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_fil # Fix up the container tag inside the deployment compose file _fixup_container_tag(deployment_dir, image) os.remove(spec_file_name) + + +@click.group() +@click.pass_context +def command(ctx): + '''manage a webapp deployment''' + + # Check that --stack wasn't supplied + if ctx.parent.obj.stack: + error_exit("--stack can't be supplied with the deploy-webapp command") + + +@command.command() +@click.option("--kube-config", help="Provide a config file for a k8s deployment") +@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") +@click.option("--deployment-dir", help="Create deployment files in this directory", required=True) +@click.option("--image", help="image to deploy", required=True) +@click.option("--url", help="url to serve", required=True) +@click.option("--env-file", help="environment file for webapp") +@click.pass_context +def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file): + '''create a deployment for the specified webapp container''' + + return create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py new file mode 100644 index 00000000..5f9e712b --- /dev/null +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -0,0 +1,267 @@ +# Copyright ©2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import json +import os +import shlex +import shutil +import sys +import tempfile + +import click + +from stack_orchestrator.deploy.webapp import deploy_webapp +from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, + build_container_image, push_container_image, + file_hash, deploy_to_k8s, publish_deployment, + hostname_for_deployment_request, generate_hostname_for_app) + + +def process_app_deployment_request( + ctx, + laconic: LaconicRegistryClient, + app_deployment_request, + deployment_record_namespace, + dns_record_namespace, + dns_suffix, + deployment_parent_dir, + kube_config, + image_registry +): + # 1. look up application + app = laconic.get_record(app_deployment_request.attributes.application, require=True) + + # 2. determine dns + requested_name = hostname_for_deployment_request(app_deployment_request, laconic) + + # HACK + if "." in requested_name: + raise Exception("Only unqualified hostnames allowed at this time.") + + fqdn = f"{requested_name}.{dns_suffix}" + container_tag = "%s:local" % app.attributes.name.replace("@", "") + + # 3. check ownership of existing dnsrecord vs this request + # TODO: Support foreign DNS + dns_crn = f"{dns_record_namespace}/{fqdn}" + dns_record = laconic.get_record(dns_crn) + if dns_record: + dns_record_owners = dns_record.owners + dns_request_owners = [] + if dns_record.request: + prev_request = laconic.get_record(dns_record.request, require=True) + dns_request_owners = prev_request.owners + + owner_match = None + + for owner in app_deployment_request.owners: + if owner in dns_request_owners or owner in dns_record_owners: + owner_match = owner + if owner_match: + print("Matched DnsRecord ownership to", owner) + else: + raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" % + (dns_record.id, app_deployment_request.id)) + + # 4. get build and runtime config from request + env_filename = None + if app_deployment_request.attributes.config and "env" in app_deployment_request.attributes.config: + env_filename = tempfile.mktemp() + with open(env_filename, 'w') as file: + for k, v in app_deployment_request.attributes.config["env"].items(): + file.write("%s=%s\n" % (k, shlex.quote(str(v)))) + + # 5. determine new or existing deployment + # a. check for deployment crn + app_deployment_crn = f"{deployment_record_namespace}/{fqdn}" + if app_deployment_request.attributes.deployment: + app_deployment_crn = app_deployment_request.attributes.deployment + if not app_deployment_crn.startswith(deployment_record_namespace): + raise Exception("Deployment CRN %s is not in a supported namespace" % app_deployment_request.attributes.deployment) + + deployment_record = laconic.get_record(app_deployment_crn) + deployment_dir = os.path.join(deployment_parent_dir, fqdn) + deployment_config_file = os.path.join(deployment_dir, "config.env") + # b. check for deployment directory (create if necessary) + if not os.path.exists(deployment_dir): + if deployment_record: + raise ("Deployment record %s exists, but not deployment dir %s. Please remove name." % + (app_deployment_crn, deployment_dir)) + print("deploy_webapp", deployment_dir) + deploy_webapp.create_deployment(ctx, deployment_dir, container_tag, + f"https://{fqdn}", kube_config, image_registry, env_filename) + elif env_filename: + shutil.copyfile(env_filename, deployment_config_file) + + needs_k8s_deploy = False + # 6. build container (if needed) + if not deployment_record or deployment_record.attributes.application != app.id: + build_container_image(app, container_tag) + push_container_image(deployment_dir) + needs_k8s_deploy = True + + # 7. update config (if needed) + if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config: + needs_k8s_deploy = True + + # 8. update k8s deployment + if needs_k8s_deploy: + print("Deploying to k8s") + deploy_to_k8s( + deployment_record, + deployment_dir, + ) + + publish_deployment( + laconic, + app, + deployment_record, + app_deployment_crn, + dns_record, + dns_crn, + deployment_dir, + app_deployment_request + ) + + +def load_known_requests(filename): + if filename and os.path.exists(filename): + return json.load(open(filename, "r")) + return {} + + +def dump_known_requests(filename, requests): + if not filename: + return + known_requests = load_known_requests(filename) + for r in requests: + known_requests[r.id] = r.createTime + json.dump(known_requests, open(filename, "w")) + + +@click.command() +@click.option("--kube-config", help="Provide a config file for a k8s deployment") +@click.option("--kube-config", help="Provide a config file for a k8s deployment") +@click.option("--laconic-config", help="Provide a config file for laconicd", required=True) +@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") +@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True) +@click.option("--request-id", help="The ApplicationDeploymentRequest to process") +@click.option("--discover", help="Discover and process all pending ApplicationDeploymentRequests", is_flag=True, default=False) +@click.option("--state-file", help="File to store state about previously seen requests.") +@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True) +@click.option("--dns-suffix", help="DNS domain to use eg, laconic.servesthe.world") +@click.option("--record-namespace-dns", help="eg, crn://laconic/dns") +@click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments") +@click.pass_context +def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, + request_id, discover, state_file, only_update_state, + dns_suffix, record_namespace_dns, record_namespace_deployments): + if request_id and discover: + print("Cannot specify both --request-id and --discover", file=sys.stderr) + sys.exit(2) + + if not request_id and not discover: + print("Must specify either --request-id or --discover", file=sys.stderr) + sys.exit(2) + + if only_update_state and not state_file: + print("--only-update-state requires --state-file", file=sys.stderr) + sys.exit(2) + + if not only_update_state: + if not record_namespace_dns or not record_namespace_deployments or not dns_suffix: + print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr) + sys.exit(2) + + laconic = LaconicRegistryClient(laconic_config) + + # Find deployment requests. + # single request + if request_id: + requests = [laconic.get_record(request_id, require=True)] + # all requests + elif discover: + requests = laconic.app_deployment_requests() + + if only_update_state: + dump_known_requests(state_file, requests) + return + + previous_requests = load_known_requests(state_file) + + # Collapse related requests. + requests.sort(key=lambda r: r.createTime) + requests.reverse() + requests_by_name = {} + for r in requests: + app = laconic.get_record(r.attributes.application) + if not app: + print("Skipping request %s, cannot locate app." % r.id) + continue + + requested_name = r.attributes.dns + if not requested_name: + requested_name = generate_hostname_for_app(app) + print("Generating name %s for request %s." % (requested_name, r.id)) + + if requested_name not in requests_by_name: + print( + "Found request %s to run application %s on %s." + % (r.id, r.attributes.application, requested_name) + ) + requests_by_name[requested_name] = r + else: + print( + "Ignoring request %s, it is superseded by %s." + % (r.id, requests_by_name[requested_name].id) + ) + + # Find deployments. + deployments = laconic.app_deployments() + deployments_by_request = {} + for d in deployments: + if d.attributes.request: + deployments_by_request[d.attributes.request] = d + + requests_to_execute = [] + for r in requests_by_name.values(): + if r.id not in deployments_by_request: + if r.id not in previous_requests: + print(f"Request {r.id} needs to processed.") + requests_to_execute.append(r) + else: + print( + f"Skipping unsatisfied request {r.id} because we have seen it before." + ) + else: + print(f"Found satisfied request {r.id} at {deployments_by_request[r.id].names[0]}") + + print("Found %d unsatisfied request(s) to process." % len(requests_to_execute)) + + for r in requests_to_execute: + try: + process_app_deployment_request( + ctx, + laconic, + r, + record_namespace_deployments, + record_namespace_dns, + dns_suffix, + deployment_parent_dir, + kube_config, + image_registry + ) + finally: + dump_known_requests(state_file, [r]) diff --git a/stack_orchestrator/deploy/webapp/util.py b/stack_orchestrator/deploy/webapp/util.py new file mode 100644 index 00000000..ebc14c3f --- /dev/null +++ b/stack_orchestrator/deploy/webapp/util.py @@ -0,0 +1,303 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import hashlib +import json +import os +import random +import subprocess +import sys +import tempfile +import uuid + +import yaml + + +class AttrDict(dict): + def __init__(self, *args, **kwargs): + super(AttrDict, self).__init__(*args, **kwargs) + self.__dict__ = self + + def __getattribute__(self, attr): + __dict__ = super(AttrDict, self).__getattribute__("__dict__") + if attr in __dict__: + v = super(AttrDict, self).__getattribute__(attr) + if isinstance(v, dict): + return AttrDict(v) + return v + + +def cmd(*vargs): + try: + result = subprocess.run(vargs, capture_output=True) + result.check_returncode() + return result.stdout.decode() + except Exception as err: + print(result.stderr.decode()) + raise err + + +class LaconicRegistryClient: + def __init__(self, config_file): + self.config_file = config_file + self.cache = AttrDict( + { + "name_or_id": {}, + } + ) + + def list_records(self, criteria={}, all=False): + args = ["laconic", "-c", self.config_file, "cns", "record", "list"] + + if all: + args.append("--all") + + if criteria: + for k, v in criteria.items(): + args.append("--%s" % k) + args.append(str(v)) + + results = [AttrDict(r) for r in json.loads(cmd(*args))] + + # Most recent records first + results.sort(key=lambda r: r.createTime) + results.reverse() + + return results + + def is_crn(self, name_or_id: str): + if name_or_id: + return str(name_or_id).startswith("crn://") + return False + + def is_id(self, name_or_id: str): + return not self.is_crn(name_or_id) + + def _add_to_cache(self, records): + if not records: + return + + for p in records: + self.cache["name_or_id"][p.id] = p + if p.names: + for crn in p.names: + self.cache["name_or_id"][crn] = p + if p.attributes.type not in self.cache: + self.cache[p.attributes.type] = [] + self.cache[p.attributes.type].append(p) + + def resolve(self, name): + if not name: + return None + + if name in self.cache.name_or_id: + return self.cache.name_or_id[name] + + args = ["laconic", "-c", self.config_file, "cns", "name", "resolve", name] + + parsed = [AttrDict(r) for r in json.loads(cmd(*args))] + if parsed: + self._add_to_cache(parsed) + return parsed[0] + + return None + + def get_record(self, name_or_id, require=False): + if not name_or_id: + if require: + raise Exception("Cannot locate record:", name_or_id) + return None + + if name_or_id in self.cache.name_or_id: + return self.cache.name_or_id[name_or_id] + + if self.is_crn(name_or_id): + return self.resolve(name_or_id) + + args = [ + "laconic", + "-c", + self.config_file, + "cns", + "record", + "get", + "--id", + name_or_id, + ] + + parsed = [AttrDict(r) for r in json.loads(cmd(*args))] + if len(parsed): + self._add_to_cache(parsed) + return parsed[0] + + if require: + raise Exception("Cannot locate record:", name_or_id) + return None + + def app_deployment_requests(self): + return self.list_records({"type": "ApplicationDeploymentRequest"}, True) + + def app_deployments(self): + return self.list_records({"type": "ApplicationDeploymentRecord"}) + + def publish(self, record, names=[]): + tmpdir = tempfile.mkdtemp() + try: + record_fname = os.path.join(tmpdir, "record.yml") + record_file = open(record_fname, 'w') + yaml.dump(record, record_file) + record_file.close() + print(open(record_fname, 'r').read()) + + new_record_id = json.loads( + cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname) + )["id"] + for name in names: + cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id) + return new_record_id + finally: + cmd("rm", "-rf", tmpdir) + + +def file_hash(filename): + return hashlib.sha1(open(filename).read().encode()).hexdigest() + + +def build_container_image(app_record, tag, extra_build_args=[]): + tmpdir = tempfile.mkdtemp() + + try: + record_id = app_record["id"] + ref = app_record.attributes.repository_ref + repo = random.choice(app_record.attributes.repository) + clone_dir = os.path.join(tmpdir, record_id) + + print(f"Cloning repository {repo} to {clone_dir} ...") + if ref: + result = subprocess.run(["git", "clone", "--depth", "1", "--branch", ref, repo, clone_dir]) + result.check_returncode() + else: + result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir]) + result.check_returncode() + + print("Building webapp ...") + build_command = [sys.argv[0], "build-webapp", "--source-repo", clone_dir, "--tag", tag] + if extra_build_args: + build_command.append("--extra-build-args") + build_command.append(" ".join(extra_build_args)) + + result = subprocess.run(build_command) + result.check_returncode() + finally: + cmd("rm", "-rf", tmpdir) + + +def push_container_image(deployment_dir): + print("Pushing image ...") + result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"]) + result.check_returncode() + + +def deploy_to_k8s(deploy_record, deployment_dir): + if not deploy_record: + command = "up" + else: + command = "update" + + result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command]) + result.check_returncode() + + +def publish_deployment(laconic: LaconicRegistryClient, + app_record, + deploy_record, + deployment_crn, + dns_record, + dns_crn, + deployment_dir, + app_deployment_request=None): + if not deploy_record: + deploy_ver = "0.0.1" + else: + deploy_ver = "0.0.%d" % (int(deploy_record.attributes.version.split(".")[-1]) + 1) + + if not dns_record: + dns_ver = "0.0.1" + else: + dns_ver = "0.0.%d" % (int(dns_record.attributes.version.split(".")[-1]) + 1) + + spec = yaml.full_load(open(os.path.join(deployment_dir, "spec.yml"))) + fqdn = spec["network"]["http-proxy"][0]["host-name"] + + uniq = uuid.uuid4() + + new_dns_record = { + "record": { + "type": "DnsRecord", + "version": dns_ver, + "name": fqdn, + "resource_type": "A", + "meta": { + "so": uniq.hex + }, + } + } + if app_deployment_request: + new_dns_record["record"]["request"] = app_deployment_request.id + + dns_id = laconic.publish(new_dns_record, [dns_crn]) + + new_deployment_record = { + "record": { + "type": "ApplicationDeploymentRecord", + "version": deploy_ver, + "url": f"https://{fqdn}", + "name": app_record.attributes.name, + "application": app_record.id, + "dns": dns_id, + "meta": { + "config": file_hash(os.path.join(deployment_dir, "config.env")), + "so": uniq.hex + }, + } + } + if app_deployment_request: + new_deployment_record["record"]["request"] = app_deployment_request.id + + deployment_id = laconic.publish(new_deployment_record, [deployment_crn]) + return {"dns": dns_id, "deployment": deployment_id} + + +def hostname_for_deployment_request(app_deployment_request, laconic): + dns_name = app_deployment_request.attributes.dns + if not dns_name: + app = laconic.get_record(app_deployment_request.attributes.application, require=True) + dns_name = generate_hostname_for_app(app) + elif dns_name.startswith("crn://"): + record = laconic.get_record(dns_name, require=True) + dns_name = record.attributes.name + return dns_name + + +def generate_hostname_for_app(app): + last_part = app.attributes.name.split("/")[-1] + m = hashlib.sha256() + m.update(app.attributes.name.encode()) + m.update(b"|") + if isinstance(app.attributes.repository, list): + m.update(app.attributes.repository[0].encode()) + else: + m.update(app.attributes.repository.encode()) + return "%s-%s" % (last_part, m.hexdigest()[0:10]) diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py index 26a011b0..fabc58ff 100644 --- a/stack_orchestrator/main.py +++ b/stack_orchestrator/main.py @@ -20,7 +20,7 @@ from stack_orchestrator.repos import setup_repositories from stack_orchestrator.build import build_containers from stack_orchestrator.build import build_npms from stack_orchestrator.build import build_webapp -from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp +from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp, deploy_webapp_from_registry from stack_orchestrator.deploy import deploy from stack_orchestrator import version from stack_orchestrator.deploy import deployment @@ -53,6 +53,7 @@ cli.add_command(build_npms.command, "build-npms") cli.add_command(build_webapp.command, "build-webapp") cli.add_command(run_webapp.command, "run-webapp") cli.add_command(deploy_webapp.command, "deploy-webapp") +cli.add_command(deploy_webapp_from_registry.command, "deploy-webapp-from-registry") cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system cli.add_command(deploy.command, "deploy-system") cli.add_command(deployment.command, "deployment")