Create new base container for pre-compiled webapps. (#507)

* Create new base container for pre-compiled webapps.

* Set the webapp dir appropriately.

app/data/compose/docker-compose-fixturenet-laconic-console.yml

@@ -3,6 +3,7 @@ services:
     restart: unless-stopped
     image: cerc/laconic-console-host:local
     environment:
+      - CERC_WEBAPP_FILES_DIR=${CERC_WEBAPP_FILES_DIR:-/usr/local/share/.config/yarn/global/node_modules/@cerc-io/console-app/dist/production}
       - LACONIC_HOSTED_ENDPOINT=${LACONIC_HOSTED_ENDPOINT:-http://localhost}
     ports:
       - "80"

app/data/container-build/cerc-laconic-console-host/Dockerfile

@@ -1,69 +1,15 @@
-# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
-# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT=18-bullseye
-FROM node:${VARIANT}
-
-ARG USERNAME=node
-ARG NPM_GLOBAL=/usr/local/share/npm-global
+FROM cerc/webapp-base:local
 
 # This container pulls npm packages from a local registry configured via these env vars
 ARG CERC_NPM_REGISTRY_URL
 ARG CERC_NPM_AUTH_TOKEN
 
-# Add NPM global to PATH.
-ENV PATH=${NPM_GLOBAL}/bin:${PATH}
-# Prevents npm from printing version warnings
-ENV NPM_CONFIG_UPDATE_NOTIFIER=false
-
-RUN \
-    # Configure global npm install location, use group to adapt to UID/GID changes
-    if ! cat /etc/group | grep -e "^npm:" > /dev/null 2>&1; then groupadd -r npm; fi \
-    && usermod -a -G npm ${USERNAME} \
-    && umask 0002 \
-    && mkdir -p ${NPM_GLOBAL} \
-    && touch /usr/local/etc/npmrc \
-    && chown ${USERNAME}:npm ${NPM_GLOBAL} /usr/local/etc/npmrc \
-    && chmod g+s ${NPM_GLOBAL} \
-    && npm config -g set prefix ${NPM_GLOBAL} \
-    && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
-    # Install eslint
-    && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
-    && npm cache clean --force > /dev/null 2>&1
-
-# [Optional] Uncomment this section to install additional OS packages.
-RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq
-
-# [Optional] Uncomment if you want to install an additional version of node using nvm
-# ARG EXTRA_NODE_VERSION=10
-# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}"
-
-# We do this to get a yq binary from the published container, for the correct architecture we're building here
-COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
-
-RUN mkdir -p /scripts
-COPY ./apply-webapp-config.sh /scripts
-COPY ./start-serving-app.sh /scripts
-
-# [Optional] Uncomment if you want to install more global node modules
-# RUN su node -c "npm install -g <your-package-list-here>"
-
 # Configure the local npm registry
 RUN npm config set @cerc-io:registry ${CERC_NPM_REGISTRY_URL} \
     && npm config set @lirewine:registry ${CERC_NPM_REGISTRY_URL} \
     && npm config set -- ${CERC_NPM_REGISTRY_URL}:_authToken ${CERC_NPM_AUTH_TOKEN}
 
-RUN mkdir -p /config
-COPY ./config.yml /config
-
-# Install simple web server for now (use nginx perhaps later)
-RUN yarn global add http-server
-
 # Globally install the payload web app package
 RUN yarn global add @cerc-io/console-app
 
-# Expose port for http
-EXPOSE 80
-
-# Default command sleeps forever so docker doesn't kill it
-CMD ["/scripts/start-serving-app.sh"]
+COPY ./config.yml /config

app/data/container-build/cerc-laconic-console-host/start-serving-app.sh

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-    set -x
-fi
-# TODO: Don't hard wire this:
-webapp_files_dir=/usr/local/share/.config/yarn/global/node_modules/@cerc-io/console-app/dist/production
-/scripts/apply-webapp-config.sh /config/config.yml ${webapp_files_dir}
-http-server -p 80 ${webapp_files_dir}

app/data/container-build/cerc-webapp-base/Dockerfile

@@ -0,0 +1,57 @@
+# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
+# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
+ARG VARIANT=18-bullseye
+FROM node:${VARIANT}
+
+ARG USERNAME=node
+ARG NPM_GLOBAL=/usr/local/share/npm-global
+
+# Add NPM global to PATH.
+ENV PATH=${NPM_GLOBAL}/bin:${PATH}
+# Prevents npm from printing version warnings
+ENV NPM_CONFIG_UPDATE_NOTIFIER=false
+
+RUN \
+    # Configure global npm install location, use group to adapt to UID/GID changes
+    if ! cat /etc/group | grep -e "^npm:" > /dev/null 2>&1; then groupadd -r npm; fi \
+    && usermod -a -G npm ${USERNAME} \
+    && umask 0002 \
+    && mkdir -p ${NPM_GLOBAL} \
+    && touch /usr/local/etc/npmrc \
+    && chown ${USERNAME}:npm ${NPM_GLOBAL} /usr/local/etc/npmrc \
+    && chmod g+s ${NPM_GLOBAL} \
+    && npm config -g set prefix ${NPM_GLOBAL} \
+    && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
+    # Install eslint
+    && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
+    && npm cache clean --force > /dev/null 2>&1
+
+# [Optional] Uncomment this section to install additional OS packages.
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -y install --no-install-recommends jq
+
+# [Optional] Uncomment if you want to install an additional version of node using nvm
+# ARG EXTRA_NODE_VERSION=10
+# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}"
+
+# We do this to get a yq binary from the published container, for the correct architecture we're building here
+COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
+
+RUN mkdir -p /scripts
+COPY ./apply-webapp-config.sh /scripts
+COPY ./start-serving-app.sh /scripts
+
+# [Optional] Uncomment if you want to install more global node modules
+# RUN su node -c "npm install -g <your-package-list-here>"
+
+RUN mkdir -p /config
+COPY ./config.yml /config
+
+# Install simple web server for now (use nginx perhaps later)
+RUN yarn global add http-server
+
+# Expose port for http
+EXPOSE 80
+
+# Default command sleeps forever so docker doesn't kill it
+CMD ["/scripts/start-serving-app.sh"]

app/data/container-build/cerc-webapp-base/apply-webapp-config.sh

@@ -18,7 +18,7 @@ if ! [[ -d ${webapp_files_dir} ]]; then
 fi
 # First some magic using yq to translate our yaml config file into an array of key value pairs like:
 # LACONIC_HOSTED_CONFIG_<path-through-objects>=<value>
-readarray -t config_kv_pair_array < <( yq '.. | select(length > 2) | ([path | join("_"), .] | join("=") )' ${config_file_name} | sed 's/^/LACONIC_HOSTED_CONFIG_/' )
+readarray -t config_kv_pair_array < <( yq '.. | ([path | join("_"), .] | join("=") )' ${config_file_name} | sort -r | sed -e '$ d' | sed 's/^/LACONIC_HOSTED_CONFIG_/' )
 declare -p config_kv_pair_array
 # Then iterate over that kv array making the template substitution in our web app files
 for kv_pair_string in "${config_kv_pair_array[@]}"

app/data/container-build/cerc-webapp-base/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Build cerc/laconic-registry-cli
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/webapp-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile ${SCRIPT_DIR}

app/data/container-build/cerc-webapp-base/config.yml

@@ -0,0 +1 @@
+# Put config here.

app/data/container-build/cerc-webapp-base/start-serving-app.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+
+CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}"
+
+/scripts/apply-webapp-config.sh /config/config.yml ${CERC_WEBAPP_FILES_DIR}
+http-server -p 80 ${CERC_WEBAPP_FILES_DIR}

app/data/container-image-list.txt

@@ -49,3 +49,4 @@ cerc/sushiswap-v3-periphery
 cerc/watcher-sushiswap
 cerc/graph-node
 cerc/sushiswap-subgraphs
+cerc/webapp-base

app/data/stacks/fixturenet-laconic-loaded/stack.yml

@@ -21,6 +21,7 @@ npms:
 containers:
   - cerc/laconicd
   - cerc/laconic-registry-cli
+  - cerc/webapp-base
   - cerc/laconic-console-host
 pods:
   - fixturenet-laconicd

app/data/stacks/mainnet-laconic/stack.yml

@@ -21,6 +21,7 @@ npms:
 containers:
   - cerc/laconicd
   - cerc/laconic-registry-cli
+  - cerc/webapp-base
   - cerc/laconic-console-host
 pods:
   - mainnet-laconicd

app/deployment_create.py

@@ -14,7 +14,6 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 import click
-from dataclasses import dataclass
 from importlib import util
 import os
 from pathlib import Path
@@ -27,6 +26,7 @@ from app.deploy_types import DeploymentContext, DeployCommandContext
 def _make_default_deployment_dir():
     return "deployment-001"
 
+
 def _get_ports(stack):
     ports = {}
     parsed_stack = get_parsed_stack_config(stack)
@@ -42,6 +42,7 @@ def _get_ports(stack):
                     ports[svc_name] = [ str(x) for x in svc["ports"] ]
     return ports
 
+
 def _get_named_volumes(stack):
     # Parse the compose files looking for named volumes
     named_volumes = []
@@ -76,30 +77,30 @@ def _create_bind_dir_if_relative(volume, path_string, compose_dir):
 
 # See: https://stackoverflow.com/questions/45699189/editing-docker-compose-yml-with-pyyaml
 def _fixup_pod_file(pod, spec, compose_dir):
-   # Fix up volumes
-   if "volumes" in spec:
-       spec_volumes = spec["volumes"]
-       if "volumes" in pod:
-           pod_volumes = pod["volumes"]
-           for volume in pod_volumes.keys():
-               if volume in spec_volumes:
-                   volume_spec = spec_volumes[volume]
-                   volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}"
-                   _create_bind_dir_if_relative(volume, volume_spec, compose_dir)
-                   new_volume_spec = {"driver": "local",
-                                      "driver_opts": {
+    # Fix up volumes
+    if "volumes" in spec:
+        spec_volumes = spec["volumes"]
+        if "volumes" in pod:
+            pod_volumes = pod["volumes"]
+            for volume in pod_volumes.keys():
+                if volume in spec_volumes:
+                    volume_spec = spec_volumes[volume]
+                    volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}"
+                    _create_bind_dir_if_relative(volume, volume_spec, compose_dir)
+                    new_volume_spec = {"driver": "local",
+                                       "driver_opts": {
                                           "type": "none",
                                           "device": volume_spec_fixedup,
                                           "o": "bind"
                                        }
-                                      }
-                   pod["volumes"][volume] = new_volume_spec
-   # Fix up ports
-   if "ports" in spec:
-       spec_ports = spec["ports"]
-       for container_name, container_ports in spec_ports.items():
-           if container_name in pod["services"]:
-               pod["services"][container_name]["ports"] = container_ports
+                                       }
+                    pod["volumes"][volume] = new_volume_spec
+    # Fix up ports
+    if "ports" in spec:
+        spec_ports = spec["ports"]
+        for container_name, container_ports in spec_ports.items():
+            if container_name in pod["services"]:
+                pod["services"][container_name]["ports"] = container_ports
 
 
 def call_stack_deploy_init(deploy_command_context):
@@ -107,10 +108,13 @@ def call_stack_deploy_init(deploy_command_context):
     # Call a function in it
     # If no function found, return None
     python_file_path = get_stack_file_path(deploy_command_context.stack).parent.joinpath("deploy", "commands.py")
-    spec = util.spec_from_file_location("commands", python_file_path)
-    imported_stack = util.module_from_spec(spec)
-    spec.loader.exec_module(imported_stack)
-    return imported_stack.init(deploy_command_context)
+    if python_file_path.exists():
+        spec = util.spec_from_file_location("commands", python_file_path)
+        imported_stack = util.module_from_spec(spec)
+        spec.loader.exec_module(imported_stack)
+        return imported_stack.init(deploy_command_context)
+    else:
+        return None
 
 
 # TODO: fold this with function above
@@ -119,10 +123,13 @@ def call_stack_deploy_setup(deploy_command_context, extra_args):
     # Call a function in it
     # If no function found, return None
     python_file_path = get_stack_file_path(deploy_command_context.stack).parent.joinpath("deploy", "commands.py")
-    spec = util.spec_from_file_location("commands", python_file_path)
-    imported_stack = util.module_from_spec(spec)
-    spec.loader.exec_module(imported_stack)
-    return imported_stack.setup(deploy_command_context, extra_args)
+    if python_file_path.exists():
+        spec = util.spec_from_file_location("commands", python_file_path)
+        imported_stack = util.module_from_spec(spec)
+        spec.loader.exec_module(imported_stack)
+        return imported_stack.setup(deploy_command_context, extra_args)
+    else:
+        return None
 
 
 # TODO: fold this with function above
@@ -131,10 +138,13 @@ def call_stack_deploy_create(deployment_context):
     # Call a function in it
     # If no function found, return None
     python_file_path = get_stack_file_path(deployment_context.command_context.stack).parent.joinpath("deploy", "commands.py")
-    spec = util.spec_from_file_location("commands", python_file_path)
-    imported_stack = util.module_from_spec(spec)
-    spec.loader.exec_module(imported_stack)
-    return imported_stack.create(deployment_context)
+    if python_file_path.exists():
+        spec = util.spec_from_file_location("commands", python_file_path)
+        imported_stack = util.module_from_spec(spec)
+        spec.loader.exec_module(imported_stack)
+        return imported_stack.create(deployment_context)
+    else:
+        return None
 
 
 # Inspect the pod yaml to find config files referenced in subdirectories