Webapp deploy (#662)

stack_orchestrator/data/compose/docker-compose-webapp-template.yml

@@ -0,0 +1,8 @@
+services:
+  webapp:
+    image: cerc/webapp-container:local
+    restart: always
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+    ports:
+      - "3000"

stack_orchestrator/data/stacks/webapp-template/README.md

@@ -0,0 +1 @@
+# Template stack for webapp deployments

stack_orchestrator/data/stacks/webapp-template/stack.yml

@@ -0,0 +1,7 @@
+version: "1.0"
+name: test
+description: "Webapp deployment stack"
+containers:
+  - cerc/webapp-template-container
+pods:
+  - webapp-template

stack_orchestrator/deploy/deploy.py

@@ -276,7 +276,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
         unique_cluster_descriptor = f"{path},{stack},{include},{exclude}"
         if ctx.debug:
             print(f"pre-hash descriptor: {unique_cluster_descriptor}")
-        hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()
+        hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()[:16]
         cluster = f"laconic-{hash}"
         if ctx.verbose:
             print(f"Using cluster name: {cluster}")

stack_orchestrator/deploy/deployment_create.py

@@ -22,6 +22,7 @@ import random
 from shutil import copy, copyfile, copytree
 import sys
 from stack_orchestrator import constants
+from stack_orchestrator.opts import opts
 from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
                                      global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
                                      get_pod_script_paths, get_plugin_code_paths, error_exit)
@@ -257,13 +258,29 @@ def _parse_config_variables(variable_values: str):
               "localhost-same, any-same, localhost-fixed-random, any-fixed-random")
 @click.pass_context
 def init(ctx, config, kube_config, image_registry, output, map_ports_to_host):
-    yaml = get_yaml()
     stack = global_options(ctx).stack
-    debug = global_options(ctx).debug
     deployer_type = ctx.obj.deployer.type
-    default_spec_file_content = call_stack_deploy_init(ctx.obj)
+    deploy_command_context = ctx.obj
+    return init_operation(
+        deploy_command_context,
+        stack, deployer_type,
+        config, kube_config,
+        image_registry,
+        output,
+        map_ports_to_host)
+
+
+# The init command's implementation is in a separate function so that we can
+# call it from other commands, bypassing the click decoration stuff
+def init_operation(deploy_command_context, stack, deployer_type, config, kube_config, image_registry, output, map_ports_to_host):
+    yaml = get_yaml()
+    default_spec_file_content = call_stack_deploy_init(deploy_command_context)
     spec_file_content = {"stack": stack, constants.deploy_to_key: deployer_type}
     if deployer_type == "k8s":
+        if kube_config is None:
+            error_exit("--kube-config must be supplied with --deploy-to k8s")
+        if image_registry is None:
+            error_exit("--image-registry must be supplied with --deploy-to k8s")
         spec_file_content.update({constants.kube_config_key: kube_config})
         spec_file_content.update({constants.image_resigtry_key: image_registry})
     else:
@@ -281,7 +298,7 @@ def init(ctx, config, kube_config, image_registry, output, map_ports_to_host):
         new_config = config_variables["config"]
         merged_config = {**new_config, **orig_config}
         spec_file_content.update({"config": merged_config})
-    if debug:
+    if opts.o.debug:
         print(f"Creating spec file for stack: {stack} with content: {spec_file_content}")
 
     ports = _get_mapped_ports(stack, map_ports_to_host)
@@ -329,12 +346,19 @@ def _copy_files_to_directory(file_paths: List[Path], directory: Path):
 @click.option("--initial-peers", help="Initial set of persistent peers")
 @click.pass_context
 def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
+    deployment_command_context = ctx.obj
+    return create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers)
+
+
+# The init command's implementation is in a separate function so that we can
+# call it from other commands, bypassing the click decoration stuff
+def create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers):
     parsed_spec = get_parsed_deployment_spec(spec_file)
     stack_name = parsed_spec["stack"]
     deployment_type = parsed_spec[constants.deploy_to_key]
     stack_file = get_stack_file_path(stack_name)
     parsed_stack = get_parsed_stack_config(stack_name)
-    if global_options(ctx).debug:
+    if opts.o.debug:
         print(f"parsed spec: {parsed_spec}")
     if deployment_dir is None:
         deployment_dir_path = _make_default_deployment_dir()
@@ -366,7 +390,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
         extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod)
         destination_pod_dir = destination_pods_dir.joinpath(pod)
         os.mkdir(destination_pod_dir)
-        if global_options(ctx).debug:
+        if opts.o.debug:
             print(f"extra config dirs: {extra_config_dirs}")
         _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir)
         with open(destination_compose_dir.joinpath("docker-compose-%s.yml" % pod), "w") as output_file:
@@ -390,7 +414,6 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
     # Delegate to the stack's Python code
     # The deploy create command doesn't require a --stack argument so we need to insert the
     # stack member here.
-    deployment_command_context = ctx.obj
     deployment_command_context.stack = stack_name
     deployment_context = DeploymentContext()
     deployment_context.init(deployment_dir_path)

stack_orchestrator/deploy/k8s/cluster_info.py

@@ -29,18 +29,19 @@ from stack_orchestrator.deploy.images import remote_tag_for_image
 class ClusterInfo:
     parsed_pod_yaml_map: Any
     image_set: Set[str] = set()
-    app_name: str = "test-app"
+    app_name: str
     environment_variables: DeployEnvVars
     spec: Spec
 
     def __init__(self) -> None:
         pass
 
-    def int(self, pod_files: List[str], compose_env_file, spec: Spec):
+    def int(self, pod_files: List[str], compose_env_file, deployment_name, spec: Spec):
         self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files)
         # Find the set of images in the pods
         self.image_set = images_for_deployment(pod_files)
         self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file))
+        self.app_name = deployment_name
         self.spec = spec
         if (opts.o.debug):
             print(f"Env vars: {self.environment_variables.map}")
@@ -67,6 +68,8 @@ class ClusterInfo:
                 proxy_to = route["proxy-to"]
                 if opts.o.debug:
                     print(f"proxy config: {path} -> {proxy_to}")
+                # proxy_to has the form <service>:<port>
+                proxy_to_port = int(proxy_to.split(":")[1])
                 paths.append(client.V1HTTPIngressPath(
                     path_type="Prefix",
                     path=path,
@@ -75,7 +78,7 @@ class ClusterInfo:
                             # TODO: this looks wrong
                             name=f"{self.app_name}-service",
                             # TODO: pull port number from the service
-                            port=client.V1ServiceBackendPort(number=80)
+                            port=client.V1ServiceBackendPort(number=proxy_to_port)
                         )
                     )
                 ))
@@ -101,14 +104,23 @@ class ClusterInfo:
             )
         return ingress
 
+    # TODO: suppoprt multiple services
     def get_service(self):
+        for pod_name in self.parsed_pod_yaml_map:
+            pod = self.parsed_pod_yaml_map[pod_name]
+            services = pod["services"]
+            for service_name in services:
+                service_info = services[service_name]
+                port = int(service_info["ports"][0])
+                if opts.o.debug:
+                    print(f"service port: {port}")
         service = client.V1Service(
             metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"),
             spec=client.V1ServiceSpec(
                 type="ClusterIP",
                 ports=[client.V1ServicePort(
-                    port=80,
+                    port=port,
-                    target_port=80
+                    target_port=port
                 )],
                 selector={"app": self.app_name}
             )
@@ -165,6 +177,10 @@ class ClusterInfo:
                 container_name = service_name
                 service_info = services[service_name]
                 image = service_info["image"]
+                port = int(service_info["ports"][0])
+                if opts.o.debug:
+                    print(f"image: {image}")
+                    print(f"service port: {port}")
                 # Re-write the image tag for remote deployment
                 image_to_use = remote_tag_for_image(
                     image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
@@ -173,7 +189,7 @@ class ClusterInfo:
                     name=container_name,
                     image=image_to_use,
                     env=envs_from_environment_variables_map(self.environment_variables.map),
-                    ports=[client.V1ContainerPort(container_port=80)],
+                    ports=[client.V1ContainerPort(container_port=port)],
                     volume_mounts=volume_mounts,
                     resources=client.V1ResourceRequirements(
                         requests={"cpu": "100m", "memory": "200Mi"},

stack_orchestrator/deploy/k8s/deploy_k8s.py

@@ -55,7 +55,7 @@ class K8sDeployer(Deployer):
         self.deployment_context = deployment_context
         self.kind_cluster_name = compose_project_name
         self.cluster_info = ClusterInfo()
-        self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec)
+        self.cluster_info.int(compose_files, compose_env_file, compose_project_name, deployment_context.spec)
         if (opts.o.debug):
             print(f"Deployment dir: {deployment_context.deployment_dir}")
             print(f"Compose files: {compose_files}")
@@ -126,6 +126,8 @@ class K8sDeployer(Deployer):
         # TODO: disable ingress for kind
         ingress: client.V1Ingress = self.cluster_info.get_ingress()
 
+        if opts.o.debug:
+            print(f"Sending this ingress: {ingress}")
         ingress_resp = self.networking_api.create_namespaced_ingress(
             namespace=self.k8s_namespace,
             body=ingress

stack_orchestrator/deploy/webapp/deploy_webapp.py

@@ -0,0 +1,113 @@
+# Copyright ©2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import click
+from pathlib import Path
+from urllib.parse import urlparse
+
+from stack_orchestrator.util import error_exit, global_options2
+from stack_orchestrator.deploy.deployment_create import init_operation, create_operation
+from stack_orchestrator.deploy.deploy import create_deploy_context
+from stack_orchestrator.deploy.deploy_types import DeployCommandContext
+
+
+def _fixup_container_tag(deployment_dir: str, image: str):
+    deployment_dir_path = Path(deployment_dir)
+    compose_file = deployment_dir_path.joinpath("compose", "docker-compose-webapp-template.yml")
+    # replace "cerc/webapp-container:local" in the file with our image tag
+    with open(compose_file) as rfile:
+        contents = rfile.read()
+        contents = contents.replace("cerc/webapp-container:local", image)
+    with open(compose_file, "w") as wfile:
+        wfile.write(contents)
+
+
+def _fixup_url_spec(spec_file_name: str, url: str):
+    # url is like: https://example.com/path
+    parsed_url = urlparse(url)
+    http_proxy_spec = f'''
+  http-proxy:
+    - host-name: {parsed_url.hostname}
+      routes:
+        - path: '{parsed_url.path if parsed_url.path else "/"}'
+          proxy-to: webapp:3000
+    '''
+    spec_file_path = Path(spec_file_name)
+    with open(spec_file_path) as rfile:
+        contents = rfile.read()
+        contents = contents + http_proxy_spec
+    with open(spec_file_path, "w") as wfile:
+        wfile.write(contents)
+
+
+@click.group()
+@click.pass_context
+def command(ctx):
+    '''manage a webapp deployment'''
+
+    # Check that --stack wasn't supplied
+    if ctx.parent.obj.stack:
+        error_exit("--stack can't be supplied with the deploy-webapp command")
+
+
+@command.command()
+@click.option("--kube-config", help="Provide a config file for a k8s deployment")
+@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster")
+@click.option("--deployment-dir", help="Create deployment files in this directory", required=True)
+@click.option("--image", help="image to deploy", required=True)
+@click.option("--url", help="url to serve", required=True)
+@click.option("--env-file", help="environment file for webapp")
+@click.pass_context
+def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
+    '''create a deployment for the specified webapp container'''
+    # Do the equivalent of:
+    # 1. laconic-so --stack webapp-template deploy --deploy-to k8s init --output webapp-spec.yml
+    #   --config (eqivalent of the contents of my-config.env)
+    # 2. laconic-so  --stack webapp-template deploy --deploy-to k8s create --deployment-dir test-deployment
+    #   --spec-file webapp-spec.yml
+    # 3. Replace the container image tag with the specified image
+    deployment_dir_path = Path(deployment_dir)
+    # Check the deployment dir does not exist
+    if deployment_dir_path.exists():
+        error_exit(f"Deployment dir {deployment_dir} already exists")
+    # Generate a temporary file name for the spec file
+    spec_file_name = "webapp-spec.yml"
+    # Specify the webapp template stack
+    stack = "webapp-template"
+    # TODO: support env file
+    deploy_command_context: DeployCommandContext = create_deploy_context(
+        global_options2(ctx), None, stack, None, None, None, env_file, "k8s"
+        )
+    init_operation(
+        deploy_command_context,
+        stack,
+        "k8s",
+        None,
+        kube_config,
+        image_registry,
+        spec_file_name,
+        None
+    )
+    # Add the TLS and DNS spec
+    _fixup_url_spec(spec_file_name, url)
+    create_operation(
+        deploy_command_context,
+        spec_file_name,
+        deployment_dir,
+        None,
+        None
+    )
+    # Fix up the container tag inside the deployment compose file
+    _fixup_container_tag(deployment_dir, image)

stack_orchestrator/deploy/webapp/run_webapp.py

@@ -22,17 +22,17 @@
 
 import hashlib
 import click
-
 from dotenv import dotenv_values
+
+from stack_orchestrator import constants
 from stack_orchestrator.deploy.deployer_factory import getDeployer
 
 
 @click.command()
 @click.option("--image", help="image to deploy", required=True)
-@click.option("--deploy-to", default="compose", help="deployment type ([Docker] 'compose' or 'k8s')")
 @click.option("--env-file", help="environment file for webapp")
 @click.pass_context
-def command(ctx, image, deploy_to, env_file):
+def command(ctx, image, env_file):
     '''build the specified webapp container'''
 
     env = {}
@@ -43,7 +43,7 @@ def command(ctx, image, deploy_to, env_file):
     hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()
     cluster = f"laconic-webapp-{hash}"
 
-    deployer = getDeployer(deploy_to,
+    deployer = getDeployer(type=constants.compose_deploy_type,
                            deployment_context=None,
                            compose_files=None,
                            compose_project_name=cluster,

stack_orchestrator/main.py

@@ -20,7 +20,7 @@ from stack_orchestrator.repos import setup_repositories
 from stack_orchestrator.build import build_containers
 from stack_orchestrator.build import build_npms
 from stack_orchestrator.build import build_webapp
-from stack_orchestrator.deploy import run_webapp
+from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp
 from stack_orchestrator.deploy import deploy
 from stack_orchestrator import version
 from stack_orchestrator.deploy import deployment
@@ -52,6 +52,7 @@ cli.add_command(build_containers.command, "build-containers")
 cli.add_command(build_npms.command, "build-npms")
 cli.add_command(build_webapp.command, "build-webapp")
 cli.add_command(run_webapp.command, "run-webapp")
+cli.add_command(deploy_webapp.command, "deploy-webapp")
 cli.add_command(deploy.command, "deploy")  # deploy is an alias for deploy-system
 cli.add_command(deploy.command, "deploy-system")
 cli.add_command(deployment.command, "deployment")