cerc-io/stack-orchestrator
17
1
Fork 2
Code Issues 112 Pull Requests 15 Actions Packages Projects Releases 473 Wiki Activity

Webapp deploy (#662)

Browse Source
This commit is contained in:
David Boreham 2023-11-27 22:02:16 -07:00 committed by GitHub
parent 1b94db27c1
commit a68cd5d65c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 191 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
stack_orchestrator/data/compose/docker-compose-webapp-template.yml Normal file
View File

@ -0,0 +1,8 @@
services:
webapp:
image: cerc/webapp-container:local
restart: always
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
ports:
- "3000"

1
stack_orchestrator/data/stacks/webapp-template/README.md Normal file
View File

@ -0,0 +1 @@
# Template stack for webapp deployments

7
stack_orchestrator/data/stacks/webapp-template/stack.yml Normal file
View File

@ -0,0 +1,7 @@
version: "1.0"
name: test
description: "Webapp deployment stack"
containers:
- cerc/webapp-template-container
pods:
- webapp-template

2
stack_orchestrator/deploy/deploy.py
View File

@ -276,7 +276,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
unique_cluster_descriptor = f"{path},{stack},{include},{exclude}"
if ctx.debug:
print(f"pre-hash descriptor: {unique_cluster_descriptor}")
hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()
hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()[:16]
cluster = f"laconic-{hash}"
if ctx.verbose:
print(f"Using cluster name: {cluster}")

37
stack_orchestrator/deploy/deployment_create.py
View File

@ -22,6 +22,7 @@ import random
from shutil import copy, copyfile, copytree
import sys
from stack_orchestrator import constants
from stack_orchestrator.opts import opts
from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
get_pod_script_paths, get_plugin_code_paths, error_exit)
@ -257,13 +258,29 @@ def _parse_config_variables(variable_values: str):
"localhost-same, any-same, localhost-fixed-random, any-fixed-random")
@click.pass_context
def init(ctx, config, kube_config, image_registry, output, map_ports_to_host):
yaml = get_yaml()
stack = global_options(ctx).stack
debug = global_options(ctx).debug
deployer_type = ctx.obj.deployer.type
default_spec_file_content = call_stack_deploy_init(ctx.obj)
deploy_command_context = ctx.obj
return init_operation(
deploy_command_context,
stack, deployer_type,
config, kube_config,
image_registry,
output,
map_ports_to_host)
# The init command's implementation is in a separate function so that we can
# call it from other commands, bypassing the click decoration stuff
def init_operation(deploy_command_context, stack, deployer_type, config, kube_config, image_registry, output, map_ports_to_host):
yaml = get_yaml()
default_spec_file_content = call_stack_deploy_init(deploy_command_context)
spec_file_content = {"stack": stack, constants.deploy_to_key: deployer_type}
if deployer_type == "k8s":
if kube_config is None:
error_exit("--kube-config must be supplied with --deploy-to k8s")
if image_registry is None:
error_exit("--image-registry must be supplied with --deploy-to k8s")
spec_file_content.update({constants.kube_config_key: kube_config})
spec_file_content.update({constants.image_resigtry_key: image_registry})
else:
@ -281,7 +298,7 @@ def init(ctx, config, kube_config, image_registry, output, map_ports_to_host):
new_config = config_variables["config"]
merged_config = {**new_config, **orig_config}
spec_file_content.update({"config": merged_config})
if debug:
if opts.o.debug:
print(f"Creating spec file for stack: {stack} with content: {spec_file_content}")
ports = _get_mapped_ports(stack, map_ports_to_host)
@ -329,12 +346,19 @@ def _copy_files_to_directory(file_paths: List[Path], directory: Path):
@click.option("--initial-peers", help="Initial set of persistent peers")
@click.pass_context
def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
deployment_command_context = ctx.obj
return create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers)
# The init command's implementation is in a separate function so that we can
# call it from other commands, bypassing the click decoration stuff
def create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers):
parsed_spec = get_parsed_deployment_spec(spec_file)
stack_name = parsed_spec["stack"]
deployment_type = parsed_spec[constants.deploy_to_key]
stack_file = get_stack_file_path(stack_name)
parsed_stack = get_parsed_stack_config(stack_name)
if global_options(ctx).debug:
if opts.o.debug:
print(f"parsed spec: {parsed_spec}")
if deployment_dir is None:
deployment_dir_path = _make_default_deployment_dir()
@ -366,7 +390,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod)
destination_pod_dir = destination_pods_dir.joinpath(pod)
os.mkdir(destination_pod_dir)
if global_options(ctx).debug:
if opts.o.debug:
print(f"extra config dirs: {extra_config_dirs}")
_fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir)
with open(destination_compose_dir.joinpath("docker-compose-%s.yml" % pod), "w") as output_file:
@ -390,7 +414,6 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
# Delegate to the stack's Python code
# The deploy create command doesn't require a --stack argument so we need to insert the
# stack member here.
deployment_command_context = ctx.obj
deployment_command_context.stack = stack_name
deployment_context = DeploymentContext()
deployment_context.init(deployment_dir_path)

28
stack_orchestrator/deploy/k8s/cluster_info.py
View File

@ -29,18 +29,19 @@ from stack_orchestrator.deploy.images import remote_tag_for_image
class ClusterInfo:
parsed_pod_yaml_map: Any
image_set: Set[str] = set()
app_name: str = "test-app"
app_name: str
environment_variables: DeployEnvVars
spec: Spec
def __init__(self) -> None:
pass
def int(self, pod_files: List[str], compose_env_file, spec: Spec):
def int(self, pod_files: List[str], compose_env_file, deployment_name, spec: Spec):
self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files)
# Find the set of images in the pods
self.image_set = images_for_deployment(pod_files)
self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file))
self.app_name = deployment_name
self.spec = spec
if (opts.o.debug):
print(f"Env vars: {self.environment_variables.map}")
@ -67,6 +68,8 @@ class ClusterInfo:
proxy_to = route["proxy-to"]
if opts.o.debug:
print(f"proxy config: {path} -> {proxy_to}")
# proxy_to has the form <service>:<port>
proxy_to_port = int(proxy_to.split(":")[1])
paths.append(client.V1HTTPIngressPath(
path_type="Prefix",
path=path,
@ -75,7 +78,7 @@ class ClusterInfo:
# TODO: this looks wrong
name=f"{self.app_name}-service",
# TODO: pull port number from the service
port=client.V1ServiceBackendPort(number=80)
port=client.V1ServiceBackendPort(number=proxy_to_port)
)
)
))
@ -101,14 +104,23 @@ class ClusterInfo:
)
return ingress
# TODO: suppoprt multiple services
def get_service(self):
for pod_name in self.parsed_pod_yaml_map:
pod = self.parsed_pod_yaml_map[pod_name]
services = pod["services"]
for service_name in services:
service_info = services[service_name]
port = int(service_info["ports"][0])
if opts.o.debug:
print(f"service port: {port}")
service = client.V1Service(
metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"),
spec=client.V1ServiceSpec(
type="ClusterIP",
ports=[client.V1ServicePort(
port=80,
target_port=80
port=port,
target_port=port
)],
selector={"app": self.app_name}
)
@ -165,6 +177,10 @@ class ClusterInfo:
container_name = service_name
service_info = services[service_name]
image = service_info["image"]
port = int(service_info["ports"][0])
if opts.o.debug:
print(f"image: {image}")
print(f"service port: {port}")
# Re-write the image tag for remote deployment
image_to_use = remote_tag_for_image(
image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
@ -173,7 +189,7 @@ class ClusterInfo:
name=container_name,
image=image_to_use,
env=envs_from_environment_variables_map(self.environment_variables.map),
ports=[client.V1ContainerPort(container_port=80)],
ports=[client.V1ContainerPort(container_port=port)],
volume_mounts=volume_mounts,
resources=client.V1ResourceRequirements(
requests={"cpu": "100m", "memory": "200Mi"},

4
stack_orchestrator/deploy/k8s/deploy_k8s.py
View File

@ -55,7 +55,7 @@ class K8sDeployer(Deployer):
self.deployment_context = deployment_context
self.kind_cluster_name = compose_project_name
self.cluster_info = ClusterInfo()
self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec)
self.cluster_info.int(compose_files, compose_env_file, compose_project_name, deployment_context.spec)
if (opts.o.debug):
print(f"Deployment dir: {deployment_context.deployment_dir}")
print(f"Compose files: {compose_files}")
@ -126,6 +126,8 @@ class K8sDeployer(Deployer):
# TODO: disable ingress for kind
ingress: client.V1Ingress = self.cluster_info.get_ingress()
if opts.o.debug:
print(f"Sending this ingress: {ingress}")
ingress_resp = self.networking_api.create_namespaced_ingress(
namespace=self.k8s_namespace,
body=ingress

0
stack_orchestrator/deploy/webapp/__init__.py Normal file
View File

113
stack_orchestrator/deploy/webapp/deploy_webapp.py Normal file
View File

@ -0,0 +1,113 @@
# Copyright ©2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import click
from pathlib import Path
from urllib.parse import urlparse
from stack_orchestrator.util import error_exit, global_options2
from stack_orchestrator.deploy.deployment_create import init_operation, create_operation
from stack_orchestrator.deploy.deploy import create_deploy_context
from stack_orchestrator.deploy.deploy_types import DeployCommandContext
def _fixup_container_tag(deployment_dir: str, image: str):
deployment_dir_path = Path(deployment_dir)
compose_file = deployment_dir_path.joinpath("compose", "docker-compose-webapp-template.yml")
# replace "cerc/webapp-container:local" in the file with our image tag
with open(compose_file) as rfile:
contents = rfile.read()
contents = contents.replace("cerc/webapp-container:local", image)
with open(compose_file, "w") as wfile:
wfile.write(contents)
def _fixup_url_spec(spec_file_name: str, url: str):
# url is like: https://example.com/path
parsed_url = urlparse(url)
http_proxy_spec = f'''
http-proxy:
- host-name: {parsed_url.hostname}
routes:
- path: '{parsed_url.path if parsed_url.path else "/"}'
proxy-to: webapp:3000
'''
spec_file_path = Path(spec_file_name)
with open(spec_file_path) as rfile:
contents = rfile.read()
contents = contents + http_proxy_spec
with open(spec_file_path, "w") as wfile:
wfile.write(contents)
@click.group()
@click.pass_context
def command(ctx):
'''manage a webapp deployment'''
# Check that --stack wasn't supplied
if ctx.parent.obj.stack:
error_exit("--stack can't be supplied with the deploy-webapp command")
@command.command()
@click.option("--kube-config", help="Provide a config file for a k8s deployment")
@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster")
@click.option("--deployment-dir", help="Create deployment files in this directory", required=True)
@click.option("--image", help="image to deploy", required=True)
@click.option("--url", help="url to serve", required=True)
@click.option("--env-file", help="environment file for webapp")
@click.pass_context
def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
'''create a deployment for the specified webapp container'''
# Do the equivalent of:
# 1. laconic-so --stack webapp-template deploy --deploy-to k8s init --output webapp-spec.yml
# --config (eqivalent of the contents of my-config.env)
# 2. laconic-so --stack webapp-template deploy --deploy-to k8s create --deployment-dir test-deployment
# --spec-file webapp-spec.yml
# 3. Replace the container image tag with the specified image
deployment_dir_path = Path(deployment_dir)
# Check the deployment dir does not exist
if deployment_dir_path.exists():
error_exit(f"Deployment dir {deployment_dir} already exists")
# Generate a temporary file name for the spec file
spec_file_name = "webapp-spec.yml"
# Specify the webapp template stack
stack = "webapp-template"
# TODO: support env file
deploy_command_context: DeployCommandContext = create_deploy_context(
global_options2(ctx), None, stack, None, None, None, env_file, "k8s"
)
init_operation(
deploy_command_context,
stack,
"k8s",
None,
kube_config,
image_registry,
spec_file_name,
None
)
# Add the TLS and DNS spec
_fixup_url_spec(spec_file_name, url)
create_operation(
deploy_command_context,
spec_file_name,
deployment_dir,
None,
None
)
# Fix up the container tag inside the deployment compose file
_fixup_container_tag(deployment_dir, image)

8
stack_orchestrator/deploy/run_webapp.py → stack_orchestrator/deploy/webapp/run_webapp.py
View File

@ -22,17 +22,17 @@
import hashlib
import click
from dotenv import dotenv_values
from stack_orchestrator import constants
from stack_orchestrator.deploy.deployer_factory import getDeployer
@click.command()
@click.option("--image", help="image to deploy", required=True)
@click.option("--deploy-to", default="compose", help="deployment type ([Docker] 'compose' or 'k8s')")
@click.option("--env-file", help="environment file for webapp")
@click.pass_context
def command(ctx, image, deploy_to, env_file):
def command(ctx, image, env_file):
'''build the specified webapp container'''
env = {}
@ -43,7 +43,7 @@ def command(ctx, image, deploy_to, env_file):
hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()
cluster = f"laconic-webapp-{hash}"
deployer = getDeployer(deploy_to,
deployer = getDeployer(type=constants.compose_deploy_type,
deployment_context=None,
compose_files=None,
compose_project_name=cluster,

3
stack_orchestrator/main.py
View File

@ -20,7 +20,7 @@ from stack_orchestrator.repos import setup_repositories
from stack_orchestrator.build import build_containers
from stack_orchestrator.build import build_npms
from stack_orchestrator.build import build_webapp
from stack_orchestrator.deploy import run_webapp
from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp
from stack_orchestrator.deploy import deploy
from stack_orchestrator import version
from stack_orchestrator.deploy import deployment
@ -52,6 +52,7 @@ cli.add_command(build_containers.command, "build-containers")
cli.add_command(build_npms.command, "build-npms")
cli.add_command(build_webapp.command, "build-webapp")
cli.add_command(run_webapp.command, "run-webapp")
cli.add_command(deploy_webapp.command, "deploy-webapp")
cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system
cli.add_command(deploy.command, "deploy-system")
cli.add_command(deployment.command, "deployment")