Webapp deploy #662

Merged
telackey merged 13 commits from dboreham/webapp-deploy into main 2023-11-28 05:02:17 +00:00
11 changed files with 191 additions and 20 deletions

View File

@ -0,0 +1,8 @@
services:
webapp:
image: cerc/webapp-container:local
restart: always
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
ports:
- "3000"

View File

@ -0,0 +1 @@
# Template stack for webapp deployments

View File

@ -0,0 +1,7 @@
version: "1.0"
name: test
description: "Webapp deployment stack"
containers:
- cerc/webapp-template-container
pods:
- webapp-template

View File

@ -276,7 +276,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
unique_cluster_descriptor = f"{path},{stack},{include},{exclude}" unique_cluster_descriptor = f"{path},{stack},{include},{exclude}"
if ctx.debug: if ctx.debug:
print(f"pre-hash descriptor: {unique_cluster_descriptor}") print(f"pre-hash descriptor: {unique_cluster_descriptor}")
hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest() hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()[:16]
cluster = f"laconic-{hash}" cluster = f"laconic-{hash}"
if ctx.verbose: if ctx.verbose:
print(f"Using cluster name: {cluster}") print(f"Using cluster name: {cluster}")

View File

@ -22,6 +22,7 @@ import random
from shutil import copy, copyfile, copytree from shutil import copy, copyfile, copytree
import sys import sys
from stack_orchestrator import constants from stack_orchestrator import constants
from stack_orchestrator.opts import opts
from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts, global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
get_pod_script_paths, get_plugin_code_paths, error_exit) get_pod_script_paths, get_plugin_code_paths, error_exit)
@ -257,13 +258,29 @@ def _parse_config_variables(variable_values: str):
"localhost-same, any-same, localhost-fixed-random, any-fixed-random") "localhost-same, any-same, localhost-fixed-random, any-fixed-random")
@click.pass_context @click.pass_context
def init(ctx, config, kube_config, image_registry, output, map_ports_to_host): def init(ctx, config, kube_config, image_registry, output, map_ports_to_host):
yaml = get_yaml()
stack = global_options(ctx).stack stack = global_options(ctx).stack
debug = global_options(ctx).debug
deployer_type = ctx.obj.deployer.type deployer_type = ctx.obj.deployer.type
default_spec_file_content = call_stack_deploy_init(ctx.obj) deploy_command_context = ctx.obj
return init_operation(
deploy_command_context,
stack, deployer_type,
config, kube_config,
image_registry,
output,
map_ports_to_host)
# The init command's implementation is in a separate function so that we can
# call it from other commands, bypassing the click decoration stuff
def init_operation(deploy_command_context, stack, deployer_type, config, kube_config, image_registry, output, map_ports_to_host):
yaml = get_yaml()
default_spec_file_content = call_stack_deploy_init(deploy_command_context)
spec_file_content = {"stack": stack, constants.deploy_to_key: deployer_type} spec_file_content = {"stack": stack, constants.deploy_to_key: deployer_type}
if deployer_type == "k8s": if deployer_type == "k8s":
if kube_config is None:
error_exit("--kube-config must be supplied with --deploy-to k8s")
if image_registry is None:
error_exit("--image-registry must be supplied with --deploy-to k8s")
spec_file_content.update({constants.kube_config_key: kube_config}) spec_file_content.update({constants.kube_config_key: kube_config})
spec_file_content.update({constants.image_resigtry_key: image_registry}) spec_file_content.update({constants.image_resigtry_key: image_registry})
else: else:
@ -281,7 +298,7 @@ def init(ctx, config, kube_config, image_registry, output, map_ports_to_host):
new_config = config_variables["config"] new_config = config_variables["config"]
merged_config = {**new_config, **orig_config} merged_config = {**new_config, **orig_config}
spec_file_content.update({"config": merged_config}) spec_file_content.update({"config": merged_config})
if debug: if opts.o.debug:
print(f"Creating spec file for stack: {stack} with content: {spec_file_content}") print(f"Creating spec file for stack: {stack} with content: {spec_file_content}")
ports = _get_mapped_ports(stack, map_ports_to_host) ports = _get_mapped_ports(stack, map_ports_to_host)
@ -329,12 +346,19 @@ def _copy_files_to_directory(file_paths: List[Path], directory: Path):
@click.option("--initial-peers", help="Initial set of persistent peers") @click.option("--initial-peers", help="Initial set of persistent peers")
@click.pass_context @click.pass_context
def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
deployment_command_context = ctx.obj
return create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers)
# The init command's implementation is in a separate function so that we can
# call it from other commands, bypassing the click decoration stuff
def create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers):
parsed_spec = get_parsed_deployment_spec(spec_file) parsed_spec = get_parsed_deployment_spec(spec_file)
stack_name = parsed_spec["stack"] stack_name = parsed_spec["stack"]
deployment_type = parsed_spec[constants.deploy_to_key] deployment_type = parsed_spec[constants.deploy_to_key]
stack_file = get_stack_file_path(stack_name) stack_file = get_stack_file_path(stack_name)
parsed_stack = get_parsed_stack_config(stack_name) parsed_stack = get_parsed_stack_config(stack_name)
if global_options(ctx).debug: if opts.o.debug:
print(f"parsed spec: {parsed_spec}") print(f"parsed spec: {parsed_spec}")
if deployment_dir is None: if deployment_dir is None:
deployment_dir_path = _make_default_deployment_dir() deployment_dir_path = _make_default_deployment_dir()
@ -366,7 +390,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod) extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod)
destination_pod_dir = destination_pods_dir.joinpath(pod) destination_pod_dir = destination_pods_dir.joinpath(pod)
os.mkdir(destination_pod_dir) os.mkdir(destination_pod_dir)
if global_options(ctx).debug: if opts.o.debug:
print(f"extra config dirs: {extra_config_dirs}") print(f"extra config dirs: {extra_config_dirs}")
_fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir) _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir)
with open(destination_compose_dir.joinpath("docker-compose-%s.yml" % pod), "w") as output_file: with open(destination_compose_dir.joinpath("docker-compose-%s.yml" % pod), "w") as output_file:
@ -390,7 +414,6 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
# Delegate to the stack's Python code # Delegate to the stack's Python code
# The deploy create command doesn't require a --stack argument so we need to insert the # The deploy create command doesn't require a --stack argument so we need to insert the
# stack member here. # stack member here.
deployment_command_context = ctx.obj
deployment_command_context.stack = stack_name deployment_command_context.stack = stack_name
deployment_context = DeploymentContext() deployment_context = DeploymentContext()
deployment_context.init(deployment_dir_path) deployment_context.init(deployment_dir_path)

View File

@ -29,18 +29,19 @@ from stack_orchestrator.deploy.images import remote_tag_for_image
class ClusterInfo: class ClusterInfo:
parsed_pod_yaml_map: Any parsed_pod_yaml_map: Any
image_set: Set[str] = set() image_set: Set[str] = set()
app_name: str = "test-app" app_name: str
environment_variables: DeployEnvVars environment_variables: DeployEnvVars
spec: Spec spec: Spec
def __init__(self) -> None: def __init__(self) -> None:
pass pass
def int(self, pod_files: List[str], compose_env_file, spec: Spec): def int(self, pod_files: List[str], compose_env_file, deployment_name, spec: Spec):
self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files) self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files)
# Find the set of images in the pods # Find the set of images in the pods
self.image_set = images_for_deployment(pod_files) self.image_set = images_for_deployment(pod_files)
self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file)) self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file))
self.app_name = deployment_name
self.spec = spec self.spec = spec
if (opts.o.debug): if (opts.o.debug):
print(f"Env vars: {self.environment_variables.map}") print(f"Env vars: {self.environment_variables.map}")
@ -67,6 +68,8 @@ class ClusterInfo:
proxy_to = route["proxy-to"] proxy_to = route["proxy-to"]
if opts.o.debug: if opts.o.debug:
print(f"proxy config: {path} -> {proxy_to}") print(f"proxy config: {path} -> {proxy_to}")
# proxy_to has the form <service>:<port>
proxy_to_port = int(proxy_to.split(":")[1])
paths.append(client.V1HTTPIngressPath( paths.append(client.V1HTTPIngressPath(
path_type="Prefix", path_type="Prefix",
path=path, path=path,
@ -75,7 +78,7 @@ class ClusterInfo:
# TODO: this looks wrong # TODO: this looks wrong
name=f"{self.app_name}-service", name=f"{self.app_name}-service",
# TODO: pull port number from the service # TODO: pull port number from the service
port=client.V1ServiceBackendPort(number=80) port=client.V1ServiceBackendPort(number=proxy_to_port)
) )
) )
)) ))
@ -101,14 +104,23 @@ class ClusterInfo:
) )
return ingress return ingress
# TODO: suppoprt multiple services
def get_service(self): def get_service(self):
for pod_name in self.parsed_pod_yaml_map:
pod = self.parsed_pod_yaml_map[pod_name]
services = pod["services"]
for service_name in services:
service_info = services[service_name]
port = int(service_info["ports"][0])
if opts.o.debug:
print(f"service port: {port}")
service = client.V1Service( service = client.V1Service(
metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"), metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"),
spec=client.V1ServiceSpec( spec=client.V1ServiceSpec(
type="ClusterIP", type="ClusterIP",
ports=[client.V1ServicePort( ports=[client.V1ServicePort(
port=80, port=port,
target_port=80 target_port=port
)], )],
selector={"app": self.app_name} selector={"app": self.app_name}
) )
@ -165,6 +177,10 @@ class ClusterInfo:
container_name = service_name container_name = service_name
service_info = services[service_name] service_info = services[service_name]
image = service_info["image"] image = service_info["image"]
port = int(service_info["ports"][0])
if opts.o.debug:
print(f"image: {image}")
print(f"service port: {port}")
# Re-write the image tag for remote deployment # Re-write the image tag for remote deployment
image_to_use = remote_tag_for_image( image_to_use = remote_tag_for_image(
image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
@ -173,7 +189,7 @@ class ClusterInfo:
name=container_name, name=container_name,
image=image_to_use, image=image_to_use,
env=envs_from_environment_variables_map(self.environment_variables.map), env=envs_from_environment_variables_map(self.environment_variables.map),
ports=[client.V1ContainerPort(container_port=80)], ports=[client.V1ContainerPort(container_port=port)],
volume_mounts=volume_mounts, volume_mounts=volume_mounts,
resources=client.V1ResourceRequirements( resources=client.V1ResourceRequirements(
requests={"cpu": "100m", "memory": "200Mi"}, requests={"cpu": "100m", "memory": "200Mi"},

View File

@ -55,7 +55,7 @@ class K8sDeployer(Deployer):
self.deployment_context = deployment_context self.deployment_context = deployment_context
self.kind_cluster_name = compose_project_name self.kind_cluster_name = compose_project_name
self.cluster_info = ClusterInfo() self.cluster_info = ClusterInfo()
self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec) self.cluster_info.int(compose_files, compose_env_file, compose_project_name, deployment_context.spec)
if (opts.o.debug): if (opts.o.debug):
print(f"Deployment dir: {deployment_context.deployment_dir}") print(f"Deployment dir: {deployment_context.deployment_dir}")
print(f"Compose files: {compose_files}") print(f"Compose files: {compose_files}")
@ -126,6 +126,8 @@ class K8sDeployer(Deployer):
# TODO: disable ingress for kind # TODO: disable ingress for kind
ingress: client.V1Ingress = self.cluster_info.get_ingress() ingress: client.V1Ingress = self.cluster_info.get_ingress()
if opts.o.debug:
print(f"Sending this ingress: {ingress}")
ingress_resp = self.networking_api.create_namespaced_ingress( ingress_resp = self.networking_api.create_namespaced_ingress(
namespace=self.k8s_namespace, namespace=self.k8s_namespace,
body=ingress body=ingress

View File

@ -0,0 +1,113 @@
# Copyright ©2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import click
from pathlib import Path
from urllib.parse import urlparse
from stack_orchestrator.util import error_exit, global_options2
from stack_orchestrator.deploy.deployment_create import init_operation, create_operation
from stack_orchestrator.deploy.deploy import create_deploy_context
from stack_orchestrator.deploy.deploy_types import DeployCommandContext
def _fixup_container_tag(deployment_dir: str, image: str):
deployment_dir_path = Path(deployment_dir)
compose_file = deployment_dir_path.joinpath("compose", "docker-compose-webapp-template.yml")
# replace "cerc/webapp-container:local" in the file with our image tag
with open(compose_file) as rfile:
contents = rfile.read()
contents = contents.replace("cerc/webapp-container:local", image)
with open(compose_file, "w") as wfile:
wfile.write(contents)
def _fixup_url_spec(spec_file_name: str, url: str):
# url is like: https://example.com/path
parsed_url = urlparse(url)
http_proxy_spec = f'''
http-proxy:
- host-name: {parsed_url.hostname}
routes:
- path: '{parsed_url.path if parsed_url.path else "/"}'
proxy-to: webapp:3000
'''
spec_file_path = Path(spec_file_name)
with open(spec_file_path) as rfile:
contents = rfile.read()
contents = contents + http_proxy_spec
with open(spec_file_path, "w") as wfile:
wfile.write(contents)
@click.group()
@click.pass_context
def command(ctx):
'''manage a webapp deployment'''
# Check that --stack wasn't supplied
if ctx.parent.obj.stack:
error_exit("--stack can't be supplied with the deploy-webapp command")
@command.command()
@click.option("--kube-config", help="Provide a config file for a k8s deployment")
@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster")
@click.option("--deployment-dir", help="Create deployment files in this directory", required=True)
@click.option("--image", help="image to deploy", required=True)
@click.option("--url", help="url to serve", required=True)
@click.option("--env-file", help="environment file for webapp")
@click.pass_context
def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
'''create a deployment for the specified webapp container'''
# Do the equivalent of:
# 1. laconic-so --stack webapp-template deploy --deploy-to k8s init --output webapp-spec.yml
# --config (eqivalent of the contents of my-config.env)
# 2. laconic-so --stack webapp-template deploy --deploy-to k8s create --deployment-dir test-deployment
# --spec-file webapp-spec.yml
# 3. Replace the container image tag with the specified image
deployment_dir_path = Path(deployment_dir)
# Check the deployment dir does not exist
if deployment_dir_path.exists():
error_exit(f"Deployment dir {deployment_dir} already exists")
# Generate a temporary file name for the spec file
spec_file_name = "webapp-spec.yml"
# Specify the webapp template stack
stack = "webapp-template"
# TODO: support env file
deploy_command_context: DeployCommandContext = create_deploy_context(
global_options2(ctx), None, stack, None, None, None, env_file, "k8s"
)
init_operation(
deploy_command_context,
stack,
"k8s",
None,
kube_config,
image_registry,
spec_file_name,
None
)
# Add the TLS and DNS spec
_fixup_url_spec(spec_file_name, url)
create_operation(
deploy_command_context,
spec_file_name,
deployment_dir,
None,
None
)
# Fix up the container tag inside the deployment compose file
_fixup_container_tag(deployment_dir, image)

View File

@ -22,17 +22,17 @@
import hashlib import hashlib
import click import click
from dotenv import dotenv_values from dotenv import dotenv_values
from stack_orchestrator import constants
from stack_orchestrator.deploy.deployer_factory import getDeployer from stack_orchestrator.deploy.deployer_factory import getDeployer
@click.command() @click.command()
@click.option("--image", help="image to deploy", required=True) @click.option("--image", help="image to deploy", required=True)
@click.option("--deploy-to", default="compose", help="deployment type ([Docker] 'compose' or 'k8s')")
@click.option("--env-file", help="environment file for webapp") @click.option("--env-file", help="environment file for webapp")
@click.pass_context @click.pass_context
def command(ctx, image, deploy_to, env_file): def command(ctx, image, env_file):
'''build the specified webapp container''' '''build the specified webapp container'''
env = {} env = {}
@ -43,7 +43,7 @@ def command(ctx, image, deploy_to, env_file):
hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest() hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()
cluster = f"laconic-webapp-{hash}" cluster = f"laconic-webapp-{hash}"
deployer = getDeployer(deploy_to, deployer = getDeployer(type=constants.compose_deploy_type,
deployment_context=None, deployment_context=None,
compose_files=None, compose_files=None,
compose_project_name=cluster, compose_project_name=cluster,

View File

@ -20,7 +20,7 @@ from stack_orchestrator.repos import setup_repositories
from stack_orchestrator.build import build_containers from stack_orchestrator.build import build_containers
from stack_orchestrator.build import build_npms from stack_orchestrator.build import build_npms
from stack_orchestrator.build import build_webapp from stack_orchestrator.build import build_webapp
from stack_orchestrator.deploy import run_webapp from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp
from stack_orchestrator.deploy import deploy from stack_orchestrator.deploy import deploy
from stack_orchestrator import version from stack_orchestrator import version
from stack_orchestrator.deploy import deployment from stack_orchestrator.deploy import deployment
@ -52,6 +52,7 @@ cli.add_command(build_containers.command, "build-containers")
cli.add_command(build_npms.command, "build-npms") cli.add_command(build_npms.command, "build-npms")
cli.add_command(build_webapp.command, "build-webapp") cli.add_command(build_webapp.command, "build-webapp")
cli.add_command(run_webapp.command, "run-webapp") cli.add_command(run_webapp.command, "run-webapp")
cli.add_command(deploy_webapp.command, "deploy-webapp")
cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system
cli.add_command(deploy.command, "deploy-system") cli.add_command(deploy.command, "deploy-system")
cli.add_command(deployment.command, "deployment") cli.add_command(deployment.command, "deployment")