Add deployment update
and deploy-webapp-from-registry
commands.
#676
@ -0,0 +1,14 @@
|
|||||||
|
record:
|
||||||
|
type: ApplicationDeploymentRecord
|
||||||
|
version: 1.2.3
|
||||||
|
name: name
|
||||||
|
description: description
|
||||||
|
application: application
|
||||||
|
url: url
|
||||||
|
dns: dns
|
||||||
|
request: request
|
||||||
|
meta:
|
||||||
|
foo: bar
|
||||||
|
tags:
|
||||||
|
- a
|
||||||
|
- b
|
@ -0,0 +1,12 @@
|
|||||||
|
record:
|
||||||
|
type: GeneralRecord
|
||||||
|
version: 1.2.3
|
||||||
|
name: name
|
||||||
|
description: description
|
||||||
|
category: category
|
||||||
|
value: value
|
||||||
|
meta:
|
||||||
|
foo: bar
|
||||||
|
tags:
|
||||||
|
- a
|
||||||
|
- b
|
@ -0,0 +1,17 @@
|
|||||||
|
record:
|
||||||
|
type: ApplicationArtifact
|
||||||
|
version: 1.2.3
|
||||||
|
name: name
|
||||||
|
description: description
|
||||||
|
application: appidgoeshere
|
||||||
|
content_type: content_type
|
||||||
|
os: os
|
||||||
|
cpu: cpu
|
||||||
|
uri:
|
||||||
|
- uri://a
|
||||||
|
- uri://b
|
||||||
|
meta:
|
||||||
|
foo: bar
|
||||||
|
tags:
|
||||||
|
- a
|
||||||
|
- b
|
@ -0,0 +1,11 @@
|
|||||||
|
record:
|
||||||
|
type: DnsRecord
|
||||||
|
version: 0.0.1
|
||||||
|
name: "foo"
|
||||||
|
resource_type: "A"
|
||||||
|
value: "bar"
|
||||||
|
meta:
|
||||||
|
foo: bar
|
||||||
|
tags:
|
||||||
|
- a
|
||||||
|
- b
|
@ -0,0 +1,18 @@
|
|||||||
|
record:
|
||||||
|
type: ApplicationRecord
|
||||||
|
version: 0.0.1
|
||||||
|
name: my-demo-app
|
||||||
|
description: "Description of my app"
|
||||||
|
homepage: http://my.demo.app
|
||||||
|
license: license
|
||||||
|
author: author
|
||||||
|
repository:
|
||||||
|
- "https://my.demo.repo"
|
||||||
|
repository_ref: "v0.1.0"
|
||||||
|
app_version: "0.1.0"
|
||||||
|
app_type: "webapp"
|
||||||
|
meta:
|
||||||
|
foo: bar
|
||||||
|
tags:
|
||||||
|
- a
|
||||||
|
- b
|
@ -0,0 +1,17 @@
|
|||||||
|
record:
|
||||||
|
type: ApplicationDeploymentRequest
|
||||||
|
version: 1.2.3
|
||||||
|
application: application
|
||||||
|
dns: dns
|
||||||
|
config:
|
||||||
|
env:
|
||||||
|
ENV_VAR_A: A
|
||||||
|
ENV_VAR_B: B
|
||||||
|
crn:
|
||||||
|
- crn://foo.bar
|
||||||
|
- crn://bar.baz
|
||||||
|
meta:
|
||||||
|
foo: bar
|
||||||
|
tags:
|
||||||
|
- a
|
||||||
|
- b
|
@ -40,6 +40,12 @@ class DockerDeployer(Deployer):
|
|||||||
except DockerException as e:
|
except DockerException as e:
|
||||||
raise DeployerException(e)
|
raise DeployerException(e)
|
||||||
|
|
||||||
|
def update(self):
|
||||||
|
try:
|
||||||
|
return self.docker.compose.restart()
|
||||||
|
except DockerException as e:
|
||||||
|
raise DeployerException(e)
|
||||||
|
|
||||||
def status(self):
|
def status(self):
|
||||||
try:
|
try:
|
||||||
for p in self.docker.compose.ps():
|
for p in self.docker.compose.ps():
|
||||||
|
@ -120,6 +120,14 @@ def status_operation(ctx):
|
|||||||
ctx.obj.deployer.status()
|
ctx.obj.deployer.status()
|
||||||
|
|
||||||
|
|
||||||
|
def update_operation(ctx):
|
||||||
|
global_context = ctx.parent.parent.obj
|
||||||
|
if not global_context.dry_run:
|
||||||
|
if global_context.verbose:
|
||||||
|
print("Running compose update")
|
||||||
|
ctx.obj.deployer.update()
|
||||||
|
|
||||||
|
|
||||||
def ps_operation(ctx):
|
def ps_operation(ctx):
|
||||||
global_context = ctx.parent.parent.obj
|
global_context = ctx.parent.parent.obj
|
||||||
if not global_context.dry_run:
|
if not global_context.dry_run:
|
||||||
|
@ -27,6 +27,10 @@ class Deployer(ABC):
|
|||||||
def down(self, timeout, volumes):
|
def down(self, timeout, volumes):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def update(self):
|
||||||
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def ps(self):
|
def ps(self):
|
||||||
pass
|
pass
|
||||||
|
@ -19,7 +19,7 @@ import sys
|
|||||||
from stack_orchestrator import constants
|
from stack_orchestrator import constants
|
||||||
from stack_orchestrator.deploy.images import push_images_operation
|
from stack_orchestrator.deploy.images import push_images_operation
|
||||||
from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation, status_operation
|
from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation, status_operation
|
||||||
from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context
|
from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context, update_operation
|
||||||
from stack_orchestrator.deploy.deploy_types import DeployCommandContext
|
from stack_orchestrator.deploy.deploy_types import DeployCommandContext
|
||||||
from stack_orchestrator.deploy.deployment_context import DeploymentContext
|
from stack_orchestrator.deploy.deployment_context import DeploymentContext
|
||||||
|
|
||||||
@ -149,3 +149,10 @@ def logs(ctx, tail, follow, extra_args):
|
|||||||
def status(ctx):
|
def status(ctx):
|
||||||
ctx.obj = make_deploy_context(ctx)
|
ctx.obj = make_deploy_context(ctx)
|
||||||
status_operation(ctx)
|
status_operation(ctx)
|
||||||
|
|
||||||
|
|
||||||
|
@command.command()
|
||||||
|
@click.pass_context
|
||||||
|
def update(ctx):
|
||||||
|
ctx.obj = make_deploy_context(ctx)
|
||||||
|
update_operation(ctx)
|
||||||
|
@ -189,6 +189,7 @@ class ClusterInfo:
|
|||||||
container = client.V1Container(
|
container = client.V1Container(
|
||||||
name=container_name,
|
name=container_name,
|
||||||
image=image_to_use,
|
image=image_to_use,
|
||||||
|
image_pull_policy="Always",
|
||||||
env=envs_from_environment_variables_map(self.environment_variables.map),
|
env=envs_from_environment_variables_map(self.environment_variables.map),
|
||||||
ports=[client.V1ContainerPort(container_port=port)],
|
ports=[client.V1ContainerPort(container_port=port)],
|
||||||
volume_mounts=volume_mounts,
|
volume_mounts=volume_mounts,
|
||||||
|
@ -13,6 +13,8 @@
|
|||||||
# You should have received a copy of the GNU Affero General Public License
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from kubernetes import client, config
|
from kubernetes import client, config
|
||||||
|
|
||||||
@ -304,6 +306,33 @@ class K8sDeployer(Deployer):
|
|||||||
log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
|
log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
|
||||||
return log_stream_from_string(log_data)
|
return log_stream_from_string(log_data)
|
||||||
|
|
||||||
|
def update(self):
|
||||||
|
self.connect_api()
|
||||||
|
ref_deployment = self.cluster_info.get_deployment()
|
||||||
|
|
||||||
|
deployment = self.apps_api.read_namespaced_deployment(
|
||||||
|
name=ref_deployment.metadata.name,
|
||||||
|
namespace=self.k8s_namespace
|
||||||
|
)
|
||||||
|
|
||||||
|
new_env = ref_deployment.spec.template.spec.containers[0].env
|
||||||
|
for container in deployment.spec.template.spec.containers:
|
||||||
|
old_env = container.env
|
||||||
|
if old_env != new_env:
|
||||||
|
container.env = new_env
|
||||||
|
|
||||||
|
deployment.spec.template.metadata.annotations = {
|
||||||
|
"kubectl.kubernetes.io/restartedAt": datetime.utcnow()
|
||||||
|
.replace(tzinfo=timezone.utc)
|
||||||
|
.isoformat()
|
||||||
|
}
|
||||||
|
|
||||||
|
self.apps_api.patch_namespaced_deployment(
|
||||||
|
name=ref_deployment.metadata.name,
|
||||||
|
namespace=self.k8s_namespace,
|
||||||
|
body=deployment
|
||||||
|
)
|
||||||
|
|
||||||
def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False):
|
def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False):
|
||||||
# We need to figure out how to do this -- check why we're being called first
|
# We need to figure out how to do this -- check why we're being called first
|
||||||
pass
|
pass
|
||||||
|
@ -54,26 +54,7 @@ def _fixup_url_spec(spec_file_name: str, url: str):
|
|||||||
wfile.write(contents)
|
wfile.write(contents)
|
||||||
|
|
||||||
|
|
||||||
@click.group()
|
def create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
|
||||||
@click.pass_context
|
|
||||||
def command(ctx):
|
|
||||||
'''manage a webapp deployment'''
|
|
||||||
|
|
||||||
# Check that --stack wasn't supplied
|
|
||||||
if ctx.parent.obj.stack:
|
|
||||||
error_exit("--stack can't be supplied with the deploy-webapp command")
|
|
||||||
|
|
||||||
|
|
||||||
@command.command()
|
|
||||||
@click.option("--kube-config", help="Provide a config file for a k8s deployment")
|
|
||||||
@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster")
|
|
||||||
@click.option("--deployment-dir", help="Create deployment files in this directory", required=True)
|
|
||||||
@click.option("--image", help="image to deploy", required=True)
|
|
||||||
@click.option("--url", help="url to serve", required=True)
|
|
||||||
@click.option("--env-file", help="environment file for webapp")
|
|
||||||
@click.pass_context
|
|
||||||
def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
|
|
||||||
'''create a deployment for the specified webapp container'''
|
|
||||||
# Do the equivalent of:
|
# Do the equivalent of:
|
||||||
# 1. laconic-so --stack webapp-template deploy --deploy-to k8s init --output webapp-spec.yml
|
# 1. laconic-so --stack webapp-template deploy --deploy-to k8s init --output webapp-spec.yml
|
||||||
# --config (eqivalent of the contents of my-config.env)
|
# --config (eqivalent of the contents of my-config.env)
|
||||||
@ -116,3 +97,27 @@ def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_fil
|
|||||||
# Fix up the container tag inside the deployment compose file
|
# Fix up the container tag inside the deployment compose file
|
||||||
_fixup_container_tag(deployment_dir, image)
|
_fixup_container_tag(deployment_dir, image)
|
||||||
os.remove(spec_file_name)
|
os.remove(spec_file_name)
|
||||||
|
|
||||||
|
|
||||||
|
@click.group()
|
||||||
|
@click.pass_context
|
||||||
|
def command(ctx):
|
||||||
|
'''manage a webapp deployment'''
|
||||||
|
|
||||||
|
# Check that --stack wasn't supplied
|
||||||
|
if ctx.parent.obj.stack:
|
||||||
|
error_exit("--stack can't be supplied with the deploy-webapp command")
|
||||||
|
|
||||||
|
|
||||||
|
@command.command()
|
||||||
|
@click.option("--kube-config", help="Provide a config file for a k8s deployment")
|
||||||
|
@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster")
|
||||||
|
@click.option("--deployment-dir", help="Create deployment files in this directory", required=True)
|
||||||
|
@click.option("--image", help="image to deploy", required=True)
|
||||||
|
@click.option("--url", help="url to serve", required=True)
|
||||||
|
@click.option("--env-file", help="environment file for webapp")
|
||||||
|
@click.pass_context
|
||||||
|
def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
|
||||||
|
'''create a deployment for the specified webapp container'''
|
||||||
|
|
||||||
|
return create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file)
|
||||||
|
267
stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py
Normal file
267
stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py
Normal file
@ -0,0 +1,267 @@
|
|||||||
|
# Copyright ©2023 Vulcanize
|
||||||
|
|
||||||
|
# This program is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shlex
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
import click
|
||||||
|
|
||||||
|
from stack_orchestrator.deploy.webapp import deploy_webapp
|
||||||
|
from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
|
||||||
|
build_container_image, push_container_image,
|
||||||
|
file_hash, deploy_to_k8s, publish_deployment,
|
||||||
|
hostname_for_deployment_request, generate_hostname_for_app)
|
||||||
|
|
||||||
|
|
||||||
|
def process_app_deployment_request(
|
||||||
|
ctx,
|
||||||
|
laconic: LaconicRegistryClient,
|
||||||
|
app_deployment_request,
|
||||||
|
deployment_record_namespace,
|
||||||
|
dns_record_namespace,
|
||||||
|
dns_suffix,
|
||||||
|
deployment_parent_dir,
|
||||||
|
kube_config,
|
||||||
|
image_registry
|
||||||
|
):
|
||||||
|
# 1. look up application
|
||||||
|
app = laconic.get_record(app_deployment_request.attributes.application, require=True)
|
||||||
|
|
||||||
|
# 2. determine dns
|
||||||
|
requested_name = hostname_for_deployment_request(app_deployment_request, laconic)
|
||||||
|
|
||||||
|
# HACK
|
||||||
|
if "." in requested_name:
|
||||||
|
raise Exception("Only unqualified hostnames allowed at this time.")
|
||||||
|
|
||||||
|
fqdn = f"{requested_name}.{dns_suffix}"
|
||||||
|
container_tag = "%s:local" % app.attributes.name.replace("@", "")
|
||||||
|
|
||||||
|
# 3. check ownership of existing dnsrecord vs this request
|
||||||
|
# TODO: Support foreign DNS
|
||||||
|
dns_crn = f"{dns_record_namespace}/{fqdn}"
|
||||||
|
dns_record = laconic.get_record(dns_crn)
|
||||||
|
if dns_record:
|
||||||
|
dns_record_owners = dns_record.owners
|
||||||
|
dns_request_owners = []
|
||||||
|
if dns_record.request:
|
||||||
|
prev_request = laconic.get_record(dns_record.request, require=True)
|
||||||
|
dns_request_owners = prev_request.owners
|
||||||
|
|
||||||
|
owner_match = None
|
||||||
|
|
||||||
|
for owner in app_deployment_request.owners:
|
||||||
|
if owner in dns_request_owners or owner in dns_record_owners:
|
||||||
|
owner_match = owner
|
||||||
|
if owner_match:
|
||||||
|
print("Matched DnsRecord ownership to", owner)
|
||||||
|
else:
|
||||||
|
raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" %
|
||||||
|
(dns_record.id, app_deployment_request.id))
|
||||||
|
|
||||||
|
# 4. get build and runtime config from request
|
||||||
|
env_filename = None
|
||||||
|
if app_deployment_request.attributes.config and "env" in app_deployment_request.attributes.config:
|
||||||
|
env_filename = tempfile.mktemp()
|
||||||
|
with open(env_filename, 'w') as file:
|
||||||
|
for k, v in app_deployment_request.attributes.config["env"].items():
|
||||||
|
file.write("%s=%s\n" % (k, shlex.quote(str(v))))
|
||||||
|
|
||||||
|
# 5. determine new or existing deployment
|
||||||
|
# a. check for deployment crn
|
||||||
|
app_deployment_crn = f"{deployment_record_namespace}/{fqdn}"
|
||||||
|
if app_deployment_request.attributes.deployment:
|
||||||
|
app_deployment_crn = app_deployment_request.attributes.deployment
|
||||||
|
if not app_deployment_crn.startswith(deployment_record_namespace):
|
||||||
|
raise Exception("Deployment CRN %s is not in a supported namespace" % app_deployment_request.attributes.deployment)
|
||||||
|
|
||||||
|
deployment_record = laconic.get_record(app_deployment_crn)
|
||||||
|
deployment_dir = os.path.join(deployment_parent_dir, fqdn)
|
||||||
|
deployment_config_file = os.path.join(deployment_dir, "config.env")
|
||||||
|
# b. check for deployment directory (create if necessary)
|
||||||
|
if not os.path.exists(deployment_dir):
|
||||||
|
if deployment_record:
|
||||||
|
raise ("Deployment record %s exists, but not deployment dir %s. Please remove name." %
|
||||||
|
(app_deployment_crn, deployment_dir))
|
||||||
|
print("deploy_webapp", deployment_dir)
|
||||||
|
deploy_webapp.create_deployment(ctx, deployment_dir, container_tag,
|
||||||
|
f"https://{fqdn}", kube_config, image_registry, env_filename)
|
||||||
|
elif env_filename:
|
||||||
|
shutil.copyfile(env_filename, deployment_config_file)
|
||||||
|
|
||||||
|
needs_k8s_deploy = False
|
||||||
|
# 6. build container (if needed)
|
||||||
|
if not deployment_record or deployment_record.attributes.application != app.id:
|
||||||
|
build_container_image(app, container_tag)
|
||||||
|
push_container_image(deployment_dir)
|
||||||
|
needs_k8s_deploy = True
|
||||||
|
|
||||||
|
# 7. update config (if needed)
|
||||||
|
if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config:
|
||||||
|
needs_k8s_deploy = True
|
||||||
|
|
||||||
|
# 8. update k8s deployment
|
||||||
|
if needs_k8s_deploy:
|
||||||
|
print("Deploying to k8s")
|
||||||
|
deploy_to_k8s(
|
||||||
|
deployment_record,
|
||||||
|
deployment_dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
publish_deployment(
|
||||||
|
laconic,
|
||||||
|
app,
|
||||||
|
deployment_record,
|
||||||
|
app_deployment_crn,
|
||||||
|
dns_record,
|
||||||
|
dns_crn,
|
||||||
|
deployment_dir,
|
||||||
|
app_deployment_request
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def load_known_requests(filename):
|
||||||
|
if filename and os.path.exists(filename):
|
||||||
|
return json.load(open(filename, "r"))
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def dump_known_requests(filename, requests):
|
||||||
|
if not filename:
|
||||||
|
return
|
||||||
|
known_requests = load_known_requests(filename)
|
||||||
|
for r in requests:
|
||||||
|
known_requests[r.id] = r.createTime
|
||||||
|
json.dump(known_requests, open(filename, "w"))
|
||||||
|
|
||||||
|
|
||||||
|
@click.command()
|
||||||
|
@click.option("--kube-config", help="Provide a config file for a k8s deployment")
|
||||||
|
@click.option("--kube-config", help="Provide a config file for a k8s deployment")
|
||||||
|
@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
|
||||||
|
@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster")
|
||||||
|
@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True)
|
||||||
|
@click.option("--request-id", help="The ApplicationDeploymentRequest to process")
|
||||||
|
@click.option("--discover", help="Discover and process all pending ApplicationDeploymentRequests", is_flag=True, default=False)
|
||||||
|
@click.option("--state-file", help="File to store state about previously seen requests.")
|
||||||
|
@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True)
|
||||||
|
@click.option("--dns-suffix", help="DNS domain to use eg, laconic.servesthe.world")
|
||||||
|
@click.option("--record-namespace-dns", help="eg, crn://laconic/dns")
|
||||||
|
@click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments")
|
||||||
|
@click.pass_context
|
||||||
|
def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir,
|
||||||
|
request_id, discover, state_file, only_update_state,
|
||||||
|
dns_suffix, record_namespace_dns, record_namespace_deployments):
|
||||||
|
if request_id and discover:
|
||||||
|
print("Cannot specify both --request-id and --discover", file=sys.stderr)
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
if not request_id and not discover:
|
||||||
|
print("Must specify either --request-id or --discover", file=sys.stderr)
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
if only_update_state and not state_file:
|
||||||
|
print("--only-update-state requires --state-file", file=sys.stderr)
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
if not only_update_state:
|
||||||
|
if not record_namespace_dns or not record_namespace_deployments or not dns_suffix:
|
||||||
|
print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr)
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
laconic = LaconicRegistryClient(laconic_config)
|
||||||
|
|
||||||
|
# Find deployment requests.
|
||||||
|
# single request
|
||||||
|
if request_id:
|
||||||
|
requests = [laconic.get_record(request_id, require=True)]
|
||||||
|
# all requests
|
||||||
|
elif discover:
|
||||||
|
requests = laconic.app_deployment_requests()
|
||||||
|
|
||||||
|
if only_update_state:
|
||||||
|
dump_known_requests(state_file, requests)
|
||||||
|
return
|
||||||
|
|
||||||
|
previous_requests = load_known_requests(state_file)
|
||||||
|
|
||||||
|
# Collapse related requests.
|
||||||
|
requests.sort(key=lambda r: r.createTime)
|
||||||
|
requests.reverse()
|
||||||
|
requests_by_name = {}
|
||||||
|
for r in requests:
|
||||||
|
app = laconic.get_record(r.attributes.application)
|
||||||
|
if not app:
|
||||||
|
print("Skipping request %s, cannot locate app." % r.id)
|
||||||
|
continue
|
||||||
|
|
||||||
|
requested_name = r.attributes.dns
|
||||||
|
if not requested_name:
|
||||||
|
requested_name = generate_hostname_for_app(app)
|
||||||
|
print("Generating name %s for request %s." % (requested_name, r.id))
|
||||||
|
|
||||||
|
if requested_name not in requests_by_name:
|
||||||
|
print(
|
||||||
|
"Found request %s to run application %s on %s."
|
||||||
|
% (r.id, r.attributes.application, requested_name)
|
||||||
|
)
|
||||||
|
requests_by_name[requested_name] = r
|
||||||
|
else:
|
||||||
|
print(
|
||||||
|
"Ignoring request %s, it is superseded by %s."
|
||||||
|
% (r.id, requests_by_name[requested_name].id)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Find deployments.
|
||||||
|
deployments = laconic.app_deployments()
|
||||||
|
deployments_by_request = {}
|
||||||
|
for d in deployments:
|
||||||
|
if d.attributes.request:
|
||||||
|
deployments_by_request[d.attributes.request] = d
|
||||||
|
|
||||||
|
requests_to_execute = []
|
||||||
|
for r in requests_by_name.values():
|
||||||
|
if r.id not in deployments_by_request:
|
||||||
|
if r.id not in previous_requests:
|
||||||
|
print(f"Request {r.id} needs to processed.")
|
||||||
|
requests_to_execute.append(r)
|
||||||
|
else:
|
||||||
|
print(
|
||||||
|
f"Skipping unsatisfied request {r.id} because we have seen it before."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
print(f"Found satisfied request {r.id} at {deployments_by_request[r.id].names[0]}")
|
||||||
|
|
||||||
|
print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))
|
||||||
|
|
||||||
|
for r in requests_to_execute:
|
||||||
|
try:
|
||||||
|
process_app_deployment_request(
|
||||||
|
ctx,
|
||||||
|
laconic,
|
||||||
|
r,
|
||||||
|
record_namespace_deployments,
|
||||||
|
record_namespace_dns,
|
||||||
|
dns_suffix,
|
||||||
|
deployment_parent_dir,
|
||||||
|
kube_config,
|
||||||
|
image_registry
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
dump_known_requests(state_file, [r])
|
303
stack_orchestrator/deploy/webapp/util.py
Normal file
303
stack_orchestrator/deploy/webapp/util.py
Normal file
@ -0,0 +1,303 @@
|
|||||||
|
# Copyright © 2023 Vulcanize
|
||||||
|
|
||||||
|
# This program is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
|
||||||
|
class AttrDict(dict):
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super(AttrDict, self).__init__(*args, **kwargs)
|
||||||
|
self.__dict__ = self
|
||||||
|
|
||||||
|
def __getattribute__(self, attr):
|
||||||
|
__dict__ = super(AttrDict, self).__getattribute__("__dict__")
|
||||||
|
if attr in __dict__:
|
||||||
|
v = super(AttrDict, self).__getattribute__(attr)
|
||||||
|
if isinstance(v, dict):
|
||||||
|
return AttrDict(v)
|
||||||
|
return v
|
||||||
|
|
||||||
|
|
||||||
|
def cmd(*vargs):
|
||||||
|
try:
|
||||||
|
result = subprocess.run(vargs, capture_output=True)
|
||||||
|
result.check_returncode()
|
||||||
|
return result.stdout.decode()
|
||||||
|
except Exception as err:
|
||||||
|
print(result.stderr.decode())
|
||||||
|
raise err
|
||||||
|
|
||||||
|
|
||||||
|
class LaconicRegistryClient:
|
||||||
|
def __init__(self, config_file):
|
||||||
|
self.config_file = config_file
|
||||||
|
self.cache = AttrDict(
|
||||||
|
{
|
||||||
|
"name_or_id": {},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def list_records(self, criteria={}, all=False):
|
||||||
|
args = ["laconic", "-c", self.config_file, "cns", "record", "list"]
|
||||||
|
|
||||||
|
if all:
|
||||||
|
args.append("--all")
|
||||||
|
|
||||||
|
if criteria:
|
||||||
|
for k, v in criteria.items():
|
||||||
|
args.append("--%s" % k)
|
||||||
|
args.append(str(v))
|
||||||
|
|
||||||
|
results = [AttrDict(r) for r in json.loads(cmd(*args))]
|
||||||
|
|
||||||
|
# Most recent records first
|
||||||
|
results.sort(key=lambda r: r.createTime)
|
||||||
|
results.reverse()
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
def is_crn(self, name_or_id: str):
|
||||||
|
if name_or_id:
|
||||||
|
return str(name_or_id).startswith("crn://")
|
||||||
|
return False
|
||||||
|
|
||||||
|
def is_id(self, name_or_id: str):
|
||||||
|
return not self.is_crn(name_or_id)
|
||||||
|
|
||||||
|
def _add_to_cache(self, records):
|
||||||
|
if not records:
|
||||||
|
return
|
||||||
|
|
||||||
|
for p in records:
|
||||||
|
self.cache["name_or_id"][p.id] = p
|
||||||
|
if p.names:
|
||||||
|
for crn in p.names:
|
||||||
|
self.cache["name_or_id"][crn] = p
|
||||||
|
if p.attributes.type not in self.cache:
|
||||||
|
self.cache[p.attributes.type] = []
|
||||||
|
self.cache[p.attributes.type].append(p)
|
||||||
|
|
||||||
|
def resolve(self, name):
|
||||||
|
if not name:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if name in self.cache.name_or_id:
|
||||||
|
return self.cache.name_or_id[name]
|
||||||
|
|
||||||
|
args = ["laconic", "-c", self.config_file, "cns", "name", "resolve", name]
|
||||||
|
|
||||||
|
parsed = [AttrDict(r) for r in json.loads(cmd(*args))]
|
||||||
|
if parsed:
|
||||||
|
self._add_to_cache(parsed)
|
||||||
|
return parsed[0]
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_record(self, name_or_id, require=False):
|
||||||
|
if not name_or_id:
|
||||||
|
if require:
|
||||||
|
raise Exception("Cannot locate record:", name_or_id)
|
||||||
|
return None
|
||||||
|
|
||||||
|
if name_or_id in self.cache.name_or_id:
|
||||||
|
return self.cache.name_or_id[name_or_id]
|
||||||
|
|
||||||
|
if self.is_crn(name_or_id):
|
||||||
|
return self.resolve(name_or_id)
|
||||||
|
|
||||||
|
args = [
|
||||||
|
"laconic",
|
||||||
|
"-c",
|
||||||
|
self.config_file,
|
||||||
|
"cns",
|
||||||
|
"record",
|
||||||
|
"get",
|
||||||
|
"--id",
|
||||||
|
name_or_id,
|
||||||
|
]
|
||||||
|
|
||||||
|
parsed = [AttrDict(r) for r in json.loads(cmd(*args))]
|
||||||
|
if len(parsed):
|
||||||
|
self._add_to_cache(parsed)
|
||||||
|
return parsed[0]
|
||||||
|
|
||||||
|
if require:
|
||||||
|
raise Exception("Cannot locate record:", name_or_id)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def app_deployment_requests(self):
|
||||||
|
return self.list_records({"type": "ApplicationDeploymentRequest"}, True)
|
||||||
|
|
||||||
|
def app_deployments(self):
|
||||||
|
return self.list_records({"type": "ApplicationDeploymentRecord"})
|
||||||
|
|
||||||
|
def publish(self, record, names=[]):
|
||||||
|
tmpdir = tempfile.mkdtemp()
|
||||||
|
try:
|
||||||
|
record_fname = os.path.join(tmpdir, "record.yml")
|
||||||
|
record_file = open(record_fname, 'w')
|
||||||
|
yaml.dump(record, record_file)
|
||||||
|
record_file.close()
|
||||||
|
print(open(record_fname, 'r').read())
|
||||||
|
|
||||||
|
new_record_id = json.loads(
|
||||||
|
cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname)
|
||||||
|
)["id"]
|
||||||
|
for name in names:
|
||||||
|
cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id)
|
||||||
|
return new_record_id
|
||||||
|
finally:
|
||||||
|
cmd("rm", "-rf", tmpdir)
|
||||||
|
|
||||||
|
|
||||||
|
def file_hash(filename):
|
||||||
|
return hashlib.sha1(open(filename).read().encode()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def build_container_image(app_record, tag, extra_build_args=[]):
|
||||||
|
tmpdir = tempfile.mkdtemp()
|
||||||
|
|
||||||
|
try:
|
||||||
|
record_id = app_record["id"]
|
||||||
|
ref = app_record.attributes.repository_ref
|
||||||
|
repo = random.choice(app_record.attributes.repository)
|
||||||
|
clone_dir = os.path.join(tmpdir, record_id)
|
||||||
|
|
||||||
|
print(f"Cloning repository {repo} to {clone_dir} ...")
|
||||||
|
if ref:
|
||||||
|
result = subprocess.run(["git", "clone", "--depth", "1", "--branch", ref, repo, clone_dir])
|
||||||
|
result.check_returncode()
|
||||||
|
else:
|
||||||
|
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir])
|
||||||
|
result.check_returncode()
|
||||||
|
|
||||||
|
print("Building webapp ...")
|
||||||
|
build_command = [sys.argv[0], "build-webapp", "--source-repo", clone_dir, "--tag", tag]
|
||||||
|
if extra_build_args:
|
||||||
|
build_command.append("--extra-build-args")
|
||||||
|
build_command.append(" ".join(extra_build_args))
|
||||||
|
|
||||||
|
result = subprocess.run(build_command)
|
||||||
|
result.check_returncode()
|
||||||
|
finally:
|
||||||
|
cmd("rm", "-rf", tmpdir)
|
||||||
|
|
||||||
|
|
||||||
|
def push_container_image(deployment_dir):
|
||||||
|
print("Pushing image ...")
|
||||||
|
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"])
|
||||||
|
result.check_returncode()
|
||||||
|
|
||||||
|
|
||||||
|
def deploy_to_k8s(deploy_record, deployment_dir):
|
||||||
|
if not deploy_record:
|
||||||
|
command = "up"
|
||||||
|
else:
|
||||||
|
command = "update"
|
||||||
|
|
||||||
|
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command])
|
||||||
|
result.check_returncode()
|
||||||
|
|
||||||
|
|
||||||
|
def publish_deployment(laconic: LaconicRegistryClient,
|
||||||
|
app_record,
|
||||||
|
deploy_record,
|
||||||
|
deployment_crn,
|
||||||
|
dns_record,
|
||||||
|
dns_crn,
|
||||||
|
deployment_dir,
|
||||||
|
app_deployment_request=None):
|
||||||
|
if not deploy_record:
|
||||||
|
deploy_ver = "0.0.1"
|
||||||
|
else:
|
||||||
|
deploy_ver = "0.0.%d" % (int(deploy_record.attributes.version.split(".")[-1]) + 1)
|
||||||
|
|
||||||
|
if not dns_record:
|
||||||
|
dns_ver = "0.0.1"
|
||||||
|
else:
|
||||||
|
dns_ver = "0.0.%d" % (int(dns_record.attributes.version.split(".")[-1]) + 1)
|
||||||
|
|
||||||
|
spec = yaml.full_load(open(os.path.join(deployment_dir, "spec.yml")))
|
||||||
|
fqdn = spec["network"]["http-proxy"][0]["host-name"]
|
||||||
|
|
||||||
|
uniq = uuid.uuid4()
|
||||||
|
|
||||||
|
new_dns_record = {
|
||||||
|
"record": {
|
||||||
|
"type": "DnsRecord",
|
||||||
|
"version": dns_ver,
|
||||||
|
"name": fqdn,
|
||||||
|
"resource_type": "A",
|
||||||
|
"meta": {
|
||||||
|
"so": uniq.hex
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if app_deployment_request:
|
||||||
|
new_dns_record["record"]["request"] = app_deployment_request.id
|
||||||
|
|
||||||
|
dns_id = laconic.publish(new_dns_record, [dns_crn])
|
||||||
|
|
||||||
|
new_deployment_record = {
|
||||||
|
"record": {
|
||||||
|
"type": "ApplicationDeploymentRecord",
|
||||||
|
"version": deploy_ver,
|
||||||
|
"url": f"https://{fqdn}",
|
||||||
|
"name": app_record.attributes.name,
|
||||||
|
"application": app_record.id,
|
||||||
|
"dns": dns_id,
|
||||||
|
"meta": {
|
||||||
|
"config": file_hash(os.path.join(deployment_dir, "config.env")),
|
||||||
|
"so": uniq.hex
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if app_deployment_request:
|
||||||
|
new_deployment_record["record"]["request"] = app_deployment_request.id
|
||||||
|
|
||||||
|
deployment_id = laconic.publish(new_deployment_record, [deployment_crn])
|
||||||
|
return {"dns": dns_id, "deployment": deployment_id}
|
||||||
|
|
||||||
|
|
||||||
|
def hostname_for_deployment_request(app_deployment_request, laconic):
|
||||||
|
dns_name = app_deployment_request.attributes.dns
|
||||||
|
if not dns_name:
|
||||||
|
app = laconic.get_record(app_deployment_request.attributes.application, require=True)
|
||||||
|
dns_name = generate_hostname_for_app(app)
|
||||||
|
elif dns_name.startswith("crn://"):
|
||||||
|
record = laconic.get_record(dns_name, require=True)
|
||||||
|
dns_name = record.attributes.name
|
||||||
|
return dns_name
|
||||||
|
|
||||||
|
|
||||||
|
def generate_hostname_for_app(app):
|
||||||
|
last_part = app.attributes.name.split("/")[-1]
|
||||||
|
m = hashlib.sha256()
|
||||||
|
m.update(app.attributes.name.encode())
|
||||||
|
m.update(b"|")
|
||||||
|
if isinstance(app.attributes.repository, list):
|
||||||
|
m.update(app.attributes.repository[0].encode())
|
||||||
|
else:
|
||||||
|
m.update(app.attributes.repository.encode())
|
||||||
|
return "%s-%s" % (last_part, m.hexdigest()[0:10])
|
@ -20,7 +20,7 @@ from stack_orchestrator.repos import setup_repositories
|
|||||||
from stack_orchestrator.build import build_containers
|
from stack_orchestrator.build import build_containers
|
||||||
from stack_orchestrator.build import build_npms
|
from stack_orchestrator.build import build_npms
|
||||||
from stack_orchestrator.build import build_webapp
|
from stack_orchestrator.build import build_webapp
|
||||||
from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp
|
from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp, deploy_webapp_from_registry
|
||||||
from stack_orchestrator.deploy import deploy
|
from stack_orchestrator.deploy import deploy
|
||||||
from stack_orchestrator import version
|
from stack_orchestrator import version
|
||||||
from stack_orchestrator.deploy import deployment
|
from stack_orchestrator.deploy import deployment
|
||||||
@ -53,6 +53,7 @@ cli.add_command(build_npms.command, "build-npms")
|
|||||||
cli.add_command(build_webapp.command, "build-webapp")
|
cli.add_command(build_webapp.command, "build-webapp")
|
||||||
cli.add_command(run_webapp.command, "run-webapp")
|
cli.add_command(run_webapp.command, "run-webapp")
|
||||||
cli.add_command(deploy_webapp.command, "deploy-webapp")
|
cli.add_command(deploy_webapp.command, "deploy-webapp")
|
||||||
|
cli.add_command(deploy_webapp_from_registry.command, "deploy-webapp-from-registry")
|
||||||
cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system
|
cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system
|
||||||
cli.add_command(deploy.command, "deploy-system")
|
cli.add_command(deploy.command, "deploy-system")
|
||||||
cli.add_command(deployment.command, "deployment")
|
cli.add_command(deployment.command, "deployment")
|
||||||
|
Loading…
Reference in New Issue
Block a user