If the next version is unsupported, print a big warning and try higher version. (#674)

* Add --port option to run-webapp

* Fixed merge

* lint

docs/webapp.md

@@ -34,7 +34,7 @@ To test locally run:
 
 ## Running
 
-With `run-webapp` a new container will be launched with runtime configuration provided by `--env-file` (if specified) and published on an available port.  Multiple instances can be launched with different configuration.
+With `run-webapp` a new container will be launched on the local machine, with runtime configuration provided by `--env-file` (if specified) and published on an available port.  Multiple instances can be launched with different configuration.
 
 **Example**:
 ```
@@ -52,3 +52,13 @@ Image: cerc/test-progressive-web-app:local
 ID: 9ab96494f563aafb6c057d88df58f9eca81b90f8721a4e068493a289a976051c
 URL: http://localhost:32769
 ```
+
+## Deploying
+
+Use the subcommand `deploy-webapp create` to make a deployment directory that can be subsequently deployed to a Kubernetes cluster.
+Example commands are shown below, assuming that the webapp container image `cerc/test-progressive-web-app:local` has already been built:
+```
+$ laconic-so deploy-webapp create --kube-config ~/kubectl/k8s-kubeconfig.yaml --image-registry registry.digitalocean.com/laconic-registry --deployment-dir webapp-k8s-deployment --image cerc/test-progressive-web-app:local --url https://test-pwa-app.hosting.laconic.com/ --env-file test-webapp.env
+$ laconic-so deployment --dir webapp-k8s-deployment push-images
+$ laconic-so deployment --dir webapp-k8s-deployment start
+```

stack_orchestrator/constants.py

@@ -14,6 +14,8 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 stack_file_name = "stack.yml"
+spec_file_name = "spec.yml"
+config_file_name = "config.env"
 compose_deploy_type = "compose"
 k8s_kind_deploy_type = "k8s-kind"
 k8s_deploy_type = "k8s"

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -24,6 +24,8 @@ RUN \
     && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
     # Install eslint
     && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
+    # Install semver
+    && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
     && npm cache clean --force > /dev/null 2>&1
 
 # [Optional] Uncomment this section to install additional OS packages.

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -4,10 +4,12 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
+CERC_MIN_NEXTVER=13.4.2
+
 CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}"
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
-  if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then
+  if [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
   else
     CERC_BUILD_TOOL=npm
@@ -101,13 +103,35 @@ cat package.dist | jq '.scripts.cerc_compile = "next experimental-compile"' | jq
 CUR_NEXT_VERSION="`jq -r '.dependencies.next' package.json`"
 
 if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VERSION" ]; then
-  echo "Changing 'next' version specifier from '$CUR_NEXT_VERSION' to '$CERC_NEXT_VERSION' (set with --build-arg CERC_NEXT_VERSION)"
+  echo "Changing 'next' version specifier from '$CUR_NEXT_VERSION' to '$CERC_NEXT_VERSION' (set with '--extra-build-args \"--build-arg CERC_NEXT_VERSION=$CERC_NEXT_VERSION\"')"
   cat package.json | jq ".dependencies.next = \"$CERC_NEXT_VERSION\"" | sponge package.json
-else
-  echo "'next' version specifier '$CUR_NEXT_VERSION' (override with --build-arg CERC_NEXT_VERSION)"
 fi
 
 $CERC_BUILD_TOOL install || exit 1
+
+CUR_NEXT_VERSION=`jq -r '.version' node_modules/next/package.json`
+
+semver -p -r ">=$CERC_MIN_NEXTVER" $CUR_NEXT_VERSION
+if [ $? -ne 0 ]; then
+  cat <<EOF
+
+###############################################################################
+
+WARNING: 'next' $CUR_NEXT_VERSION < minimum version $CERC_MIN_NEXTVER.
+
+Attempting to build with '^$CERC_MIN_NEXTVER'.  If this fails, you should upgrade
+the dependency in your webapp, or specify an explicit 'next' version
+to use for the build with:
+
+     --extra-build-args "--build-arg CERC_NEXT_VERSION=<version>"
+
+###############################################################################
+
+EOF
+  cat package.json | jq ".dependencies.next = \"^$CERC_MIN_NEXTVER\"" | sponge package.json
+  $CERC_BUILD_TOOL install || exit 1
+fi
+
 $CERC_BUILD_TOOL run cerc_compile || exit 1
 
 exit 0

stack_orchestrator/deploy/compose/deploy_docker.py

@@ -64,10 +64,10 @@ class DockerDeployer(Deployer):
         except DockerException as e:
             raise DeployerException(e)
 
-    def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False):
+    def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False):
         try:
             return self.docker.run(image=image, command=command, user=user, volumes=volumes,
-                                   entrypoint=entrypoint, envs=env, detach=detach, publish_all=True)
+                                   entrypoint=entrypoint, envs=env, detach=detach, publish=ports, publish_all=len(ports) == 0)
         except DockerException as e:
             raise DeployerException(e)
 

stack_orchestrator/deploy/deployer.py

@@ -44,7 +44,7 @@ class Deployer(ABC):
         pass
 
     @abstractmethod
-    def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False):
+    def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False):
         pass
 
 

stack_orchestrator/deploy/deployment_context.py

@@ -16,6 +16,7 @@
 
 from pathlib import Path
 
+from stack_orchestrator import constants
 from stack_orchestrator.deploy.stack import Stack
 from stack_orchestrator.deploy.spec import Spec
 
@@ -26,13 +27,13 @@ class DeploymentContext:
     stack: Stack
 
     def get_stack_file(self):
-        return self.deployment_dir.joinpath("stack.yml")
+        return self.deployment_dir.joinpath(constants.stack_file_name)
 
     def get_spec_file(self):
-        return self.deployment_dir.joinpath("spec.yml")
+        return self.deployment_dir.joinpath(constants.spec_file_name)
 
     def get_env_file(self):
-        return self.deployment_dir.joinpath("config.env")
+        return self.deployment_dir.joinpath(constants.config_file_name)
 
     # TODO: implement me
     def get_cluster_name(self):

stack_orchestrator/deploy/deployment_create.py

@@ -25,7 +25,7 @@ from stack_orchestrator import constants
 from stack_orchestrator.opts import opts
 from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
                                      global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
-                                     get_pod_script_paths, get_plugin_code_paths, error_exit)
+                                     get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file)
 from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand
 from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
@@ -244,12 +244,13 @@ def _parse_config_variables(variable_values: str):
                 variable_name = variable_value_pair[0]
                 variable_value = variable_value_pair[1]
                 result_values[variable_name] = variable_value
-            result = {"config": result_values}
+            result = result_values
     return result
 
 
 @click.command()
 @click.option("--config", help="Provide config variables for the deployment")
+@click.option("--config-file", help="Provide config variables in a file for the deployment")
 @click.option("--kube-config", help="Provide a config file for a k8s deployment")
 @click.option("--image-registry", help="Provide a container image registry url for this k8s cluster")
 @click.option("--output", required=True, help="Write yaml spec file here")
@@ -257,14 +258,15 @@ def _parse_config_variables(variable_values: str):
               help="Map ports to the host as one of: any-variable-random (default), "
               "localhost-same, any-same, localhost-fixed-random, any-fixed-random")
 @click.pass_context
-def init(ctx, config, kube_config, image_registry, output, map_ports_to_host):
+def init(ctx, config, config_file, kube_config, image_registry, output, map_ports_to_host):
     stack = global_options(ctx).stack
     deployer_type = ctx.obj.deployer.type
     deploy_command_context = ctx.obj
     return init_operation(
         deploy_command_context,
         stack, deployer_type,
-        config, kube_config,
+        config, config_file,
+        kube_config,
         image_registry,
         output,
         map_ports_to_host)
@@ -272,7 +274,8 @@ def init(ctx, config, kube_config, image_registry, output, map_ports_to_host):
 
 # The init command's implementation is in a separate function so that we can
 # call it from other commands, bypassing the click decoration stuff
-def init_operation(deploy_command_context, stack, deployer_type, config, kube_config, image_registry, output, map_ports_to_host):
+def init_operation(deploy_command_context, stack, deployer_type, config,
+                   config_file, kube_config, image_registry, output, map_ports_to_host):
     yaml = get_yaml()
     default_spec_file_content = call_stack_deploy_init(deploy_command_context)
     spec_file_content = {"stack": stack, constants.deploy_to_key: deployer_type}
@@ -292,12 +295,22 @@ def init_operation(deploy_command_context, stack, deployer_type, config, kube_co
     if default_spec_file_content:
         spec_file_content.update(default_spec_file_content)
     config_variables = _parse_config_variables(config)
+    # Implement merge, since update() overwrites
     if config_variables:
-        # Implement merge, since update() overwrites
         orig_config = spec_file_content.get("config", {})
-        new_config = config_variables["config"]
+        new_config = config_variables
         merged_config = {**new_config, **orig_config}
         spec_file_content.update({"config": merged_config})
+    if config_file:
+        config_file_path = Path(config_file)
+        if not config_file_path.exists():
+            error_exit(f"config file: {config_file} does not exist")
+        config_file_variables = env_var_map_from_file(config_file_path)
+        if config_file_variables:
+            orig_config = spec_file_content.get("config", {})
+            new_config = config_file_variables
+            merged_config = {**new_config, **orig_config}
+            spec_file_content.update({"config": merged_config})
     if opts.o.debug:
         print(f"Creating spec file for stack: {stack} with content: {spec_file_content}")
 
@@ -368,10 +381,10 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
         error_exit(f"{deployment_dir_path} already exists")
     os.mkdir(deployment_dir_path)
     # Copy spec file and the stack file into the deployment dir
-    copyfile(spec_file, deployment_dir_path.joinpath("spec.yml"))
+    copyfile(spec_file, deployment_dir_path.joinpath(constants.spec_file_name))
     copyfile(stack_file, deployment_dir_path.joinpath(os.path.basename(stack_file)))
     # Copy any config varibles from the spec file into an env file suitable for compose
-    _write_config_file(spec_file, deployment_dir_path.joinpath("config.env"))
+    _write_config_file(spec_file, deployment_dir_path.joinpath(constants.config_file_name))
     # Copy any k8s config file into the deployment dir
     if deployment_type == "k8s":
         _write_kube_config_file(Path(parsed_spec[constants.kube_config_key]),

stack_orchestrator/deploy/k8s/cluster_info.py

@@ -17,9 +17,10 @@ from kubernetes import client
 from typing import Any, List, Set
 
 from stack_orchestrator.opts import opts
+from stack_orchestrator.util import env_var_map_from_file
 from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files
 from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path
-from stack_orchestrator.deploy.k8s.helpers import env_var_map_from_file, envs_from_environment_variables_map
+from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment
 from stack_orchestrator.deploy.deploy_types import DeployEnvVars
 from stack_orchestrator.deploy.spec import Spec

stack_orchestrator/deploy/k8s/deploy_k8s.py

@@ -230,7 +230,7 @@ class K8sDeployer(Deployer):
         log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
         return log_stream_from_string(log_data)
 
-    def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False):
+    def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False):
         # We need to figure out how to do this -- check why we're being called first
         pass
 

stack_orchestrator/deploy/k8s/helpers.py

@@ -14,7 +14,6 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from kubernetes import client
-from dotenv import dotenv_values
 import os
 from pathlib import Path
 import subprocess
@@ -224,7 +223,3 @@ def generate_kind_config(deployment_dir: Path):
         f"{port_mappings_yml}\n"
         f"{mounts_yml}\n"
     )
-
-
-def env_var_map_from_file(file: Path) -> Mapping[str, str]:
-    return dotenv_values(file)

stack_orchestrator/deploy/webapp/deploy_webapp.py

@@ -98,6 +98,7 @@ def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_fil
         stack,
         "k8s",
         None,
+        env_file,
         kube_config,
         image_registry,
         spec_file_name,

stack_orchestrator/deploy/webapp/run_webapp.py

@@ -27,13 +27,16 @@ from dotenv import dotenv_values
 from stack_orchestrator import constants
 from stack_orchestrator.deploy.deployer_factory import getDeployer
 
+WEBAPP_PORT = 3000
+
 
 @click.command()
 @click.option("--image", help="image to deploy", required=True)
 @click.option("--env-file", help="environment file for webapp")
+@click.option("--port", help="port to use (default random)")
 @click.pass_context
-def command(ctx, image, env_file):
-    '''build the specified webapp container'''
+def command(ctx, image, env_file, port):
+    '''run the specified webapp container'''
 
     env = {}
     if env_file:
@@ -49,10 +52,13 @@ def command(ctx, image, env_file):
                            compose_project_name=cluster,
                            compose_env_file=None)
 
-    container = deployer.run(image, command=[], user=None, volumes=[], entrypoint=None, env=env, detach=True)
+    ports = []
+    if port:
+        ports = [(port, WEBAPP_PORT)]
+    container = deployer.run(image, command=[], user=None, volumes=[], entrypoint=None, env=env, ports=ports, detach=True)
 
     # Make configurable?
-    webappPort = "3000/tcp"
+    webappPort = f"{WEBAPP_PORT}/tcp"
     # TODO: This assumes a Docker container object...
     if webappPort in container.network_settings.ports:
         mapping = container.network_settings.ports[webappPort][0]

stack_orchestrator/util.py

@@ -18,6 +18,8 @@ import os.path
 import sys
 import ruamel.yaml
 from pathlib import Path
+from dotenv import dotenv_values
+from typing import Mapping
 
 
 def include_exclude_check(s, include, exclude):
@@ -178,3 +180,7 @@ def global_options2(ctx):
 def error_exit(s):
     print(f"ERROR: {s}")
     sys.exit(1)
+
+
+def env_var_map_from_file(file: Path) -> Mapping[str, str]:
+    return dotenv_values(file)