Add webapp doc (#652)

This fixes three issues:

1. #644 (build output)
2. #646 (error on startup)
3. automatic env quote handling (related to 2)


For the build output we now have:

```
#################################################################

Built host container for /home/telackey/tmp/iglootools-home with tag:

    cerc/iglootools-home:local

To test locally run:

    docker run -p 3000:3000 cerc/iglootools-home:local
```

For the startup error, it was hung waiting for the "success" message from the next generate output (itself a workaround for a nextjs bug fixed by this PR we submitted: https://github.com/vercel/next.js/pull/58276).

I added a timeout which will cause it to wait up to a maximum _n_ seconds before issuing:

```
ERROR: 'npm run cerc_generate' exceeded CERC_MAX_GENERATE_TIME.
```

On the quoting itself, I plan on adding a new run-webapp command, but I realized I had a decent spot to do effect the quote replacement on-the-fly after all when I am already escaping the values for insertion/replacement into JS.

The "dequoting" can be disabled with `CERC_RETAIN_ENV_QUOTES=true`.

docs/webapp.md

@@ -0,0 +1,42 @@
+### Building and Running Webapps
+
+It is possible to build and run webapps using the `build-webapp` and `run-webapp` subcommand.
+
+To make it easier to build once, and deploy to with varying configuration, compilation and static
+page generation are separated in the `build-webapp` and `run-webapp` steps, and the use of the
+environment variables via `process.env` is detected at compile-time and placeholder substituted
+which will be filled in at runtime.
+
+This offers much more flexibilty in configuration and deployment than standard build methods.
+
+## Build
+
+```
+$ cd ~/cerc
+$ git clone git@git.vdb.to:cerc-io/test-progressive-web-app.git
+$ laconic-so build-webapp --source-repo ~/cerc/test-progressive-web-app
+...
+Successfully tagged cerc/test-progressive-web-app:local
+
+
+#################################################################
+
+Built host container for ~/cerc/test-progressive-web-app with tag:
+
+    cerc/test-progressive-web-app:local
+
+To test locally run:
+
+    laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment.env
+
+```
+
+## Run
+
+```
+$ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file ~/tmp/env.igloo
+
+Image: cerc/test-progressive-web-app:local
+ID: 4c6e893bf436b3e91a2b92ce37e30e499685131705700bd92a90d2eb14eefd05
+URL: http://localhost:32768
+```

stack_orchestrator/build/build_containers.py

@@ -27,7 +27,7 @@ import subprocess
 import click
 import importlib.resources
 from pathlib import Path
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config
+from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external
 from stack_orchestrator.base import get_npm_registry_url
 
 # TODO: find a place for this
@@ -58,7 +58,8 @@ def make_container_build_env(dev_root_path: str,
     return container_build_env
 
 
-def process_container(container,
+def process_container(stack: str,
+                      container,
                       container_build_dir: str,
                       container_build_env: dict,
                       dev_root_path: str,
@@ -69,12 +70,29 @@ def process_container(container,
                       ):
     if not quiet:
         print(f"Building: {container}")
-    build_dir = os.path.join(container_build_dir, container.replace("/", "-"))
-    build_script_filename = os.path.join(build_dir, "build.sh")
+
+    default_container_tag = f"{container}:local"
+    container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
+
+    # Check if this is in an external stack
+    if stack_is_external(stack):
+        container_parent_dir = Path(stack).joinpath("container-build")
+        temp_build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
+        temp_build_script_filename = temp_build_dir.joinpath("build.sh")
+        # Now check if the container exists in the external stack.
+        if not temp_build_script_filename.exists():
+            # If not, revert to building an internal container
+            container_parent_dir = container_build_dir
+    else:
+        container_parent_dir = container_build_dir
+
+    build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
+    build_script_filename = build_dir.joinpath("build.sh")
+
     if verbose:
         print(f"Build script filename: {build_script_filename}")
     if os.path.exists(build_script_filename):
-        build_command = build_script_filename
+        build_command = build_script_filename.as_posix()
     else:
         if verbose:
             print(f"No script file found: {build_script_filename}, using default build script")
@@ -84,7 +102,7 @@ def process_container(container,
         repo_full_path = os.path.join(dev_root_path, repo_dir)
         repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
         build_command = os.path.join(container_build_dir,
-                                     "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}"
+                                     "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
     if not dry_run:
         if verbose:
             print(f"Executing: {build_command} with environment: {container_build_env}")
@@ -158,7 +176,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
 
     for container in containers_in_scope:
         if include_exclude_check(container, include, exclude):
-            process_container(container, container_build_dir, container_build_env,
+            process_container(stack, container, container_build_dir, container_build_env,
                               dev_root_path, quiet, verbose, dry_run, continue_on_error)
         else:
             if verbose:

stack_orchestrator/build/build_webapp.py

@@ -60,7 +60,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
     container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
                                                                     force_rebuild, extra_build_args)
 
-    build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet,
+    build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
                                        verbose, dry_run, continue_on_error)
 
 
@@ -73,5 +73,5 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
     webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
     container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
 
-    build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet,
+    build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
                                        verbose, dry_run, continue_on_error)

stack_orchestrator/constants.py

@@ -0,0 +1,16 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+stack_file_name = "stack.yml"

stack_orchestrator/data/compose/docker-compose-kubo.yml

@@ -2,7 +2,7 @@ version: "3.2"
 # See: https://docs.ipfs.tech/install/run-ipfs-inside-docker/#set-up
 services:
   ipfs:
-    image: ipfs/kubo:v0.24.0
+    image: ipfs/kubo:master-2023-02-20-714a968
     restart: always
     volumes:
       - ./ipfs/import:/import

stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml

@@ -1,8 +0,0 @@
-version: "3.2"
-
-services:
-  osmosis-front-end:
-    image: cerc/osmosis-front-end:local
-    restart: always
-    ports:
-      - "3002:3002" #TODO make `3000` when using the deployment feature

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -39,4 +39,4 @@ EXPOSE 3000
 COPY /scripts /scripts
 
 # Default command sleeps forever so docker doesn't kill it
-CMD ["/scripts/start-serving-app.sh"]
+ENTRYPOINT ["/scripts/start-serving-app.sh"]

stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh

@@ -11,3 +11,19 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D
 CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/nextjs-base:local}
 
 docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
+
+if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then
+  cat <<EOF
+
+#################################################################
+
+Built host container for $CERC_CONTAINER_BUILD_WORK_DIR with tag:
+
+    $CERC_CONTAINER_BUILD_TAG
+
+To test locally run:
+
+    laconic-so run-webapp --image $CERC_CONTAINER_BUILD_TAG --env-file /path/to/environment.env
+
+EOF
+fi

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh

@@ -38,6 +38,9 @@ for f in $(find "$TRG_DIR" -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'
     orig_name=$(echo -n "${e}" | sed 's/"//g')
     cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
     cur_val=$(echo -n "\$${cur_name}" | envsubst)
+    if [ "$CERC_RETAIN_ENV_QUOTES" != "true" ]; then
+      cur_val=$(sed "s/^[\"']//" <<< "$cur_val" | sed "s/[\"']//")
+    fi
     esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val")
     echo "$f: $cur_name=$cur_val"
     sed -i "s/$orig_name/$esc_val/g" $f

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -3,7 +3,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
+
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-60}
+tpid=""
+
+ctrl_c() {
+    kill $tpid $(ps -ef | grep node | grep next | awk '{print $2}') 2>/dev/null
+}
+
+trap ctrl_c INT
 
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
@@ -25,18 +34,27 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
   jq -e '.scripts.cerc_generate' package.json >/dev/null
   if [ $? -eq 0 ]; then
     npm run cerc_generate > gen.out 2>&1 &
-    tail -n0 -f gen.out | sed '/rendered as static HTML/ q'
+    tail -f gen.out &
+    tpid=$!
+
     count=0
-    while [ $count -lt 10 ]; do
+    generate_done="false"
+    while [ $count -lt $CERC_MAX_GENERATE_TIME ] && [ "$generate_done" == "false" ]; do
       sleep 1
-      ps -ef | grep 'node' | grep 'next' | grep 'generate' >/dev/null
-      if [ $? -ne 0 ]; then 
-        break
-      else
-        count=$((count + 1))
+      count=$((count + 1))
+      grep 'rendered as static HTML' gen.out > /dev/null
+      if [ $? -eq 0 ]; then
+        generate_done="true"
       fi
     done
-    kill $(ps -ef |grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
+
+    if [ $generate_done != "true" ]; then
+      echo "ERROR: 'npm run cerc_generate' not successful within CERC_MAX_GENERATE_TIME" 1>&2
+      exit 1
+    fi
+
+    kill $tpid $(ps -ef | grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
+    tpid=""
   fi
 fi
 

stack_orchestrator/data/container-build/cerc-osmosis-front-end/build.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-# Build the osmosis front end image
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-docker build -t cerc/osmosis-front-end:local -f ${CERC_REPO_BASE_DIR}/osmosis-frontend/docker/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/osmosis-frontend

stack_orchestrator/data/stacks/osmosis/README.md

@@ -1,35 +0,0 @@
-# self-hosted osmosis
-
-Build and deploy:
-- 1) self-hosted gitea,
-- 2) an ipfs node,
-- 3) the osmosis front end,
-- 4) a laconicd chain
-
-
-```
-# support image for the gitea package registry
-laconic-so --stack build-support build-containers
-
-# todo: pre-run clone
-
-# clones and builds several things
-laconic-so --stack osmosis setup-repositories
-laconic-so --stack osmosis build-containers 
-laconic-so --stack osmosis deploy up
-```
-
-Setup a test chain:
-```
-export CERC_NPM_REGISTRY_URL=https://git.vdb.to/api/packages/cerc-io/npm/
-
-laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
-
-laconic-so --stack fixturenet-laconic-loaded build-containers
-
-export LACONIC_HOSTED_ENDPOINT=http://<your-IP>
-
-laconic-so --stack fixturenet-laconic-loaded deploy up
-```
-
-then `docker exec` into the `laconicd` container and either export the private key or create a new one and send funds to it. Use that private key for `LACONIC_HOTWALLET_KEY`.

stack_orchestrator/data/stacks/osmosis/stack.yml

@@ -1,27 +0,0 @@
-version: "0.1"
-name: osmosis
-repos:
-  # these are for gitea
-  - git.vdb.to/cerc-io/hosting@names-for-so
-  - gitea.com/gitea/act_runner
-  # add the osmosis FE
-  - github.com/osmosis-labs/osmosis-frontend
-containers:
-  - cerc/act-runner
-  - cerc/act-runner-task-executor
-  # note: osmosis builds but doesn't run
-  - cerc/osmosis-front-end
-pods:
-  - name: gitea
-    repository: cerc-io/hosting
-    path: gitea
-    pre_start_command: "run-this-first.sh"
-    post_start_command: "initialize-gitea.sh"
-    # todo, e.g., mirroring all of osmosis repos: https://git.vdb.to/cerc-io/hosting/pulls/42
-  - name: act-runner
-    repository: cerc-io/hosting
-    path: act-runner
-    pre_start_command: "pre_start.sh"
-    post_start_command: "post_start.sh"
-  - osmosis-front-end
-  - kubo

stack_orchestrator/deploy/compose/deploy_docker.py

@@ -61,9 +61,10 @@ class DockerDeployer(Deployer):
         except DockerException as e:
             raise DeployerException(e)
 
-    def run(self, image, command, user, volumes, entrypoint=None):
+    def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False):
         try:
-            return self.docker.run(image=image, command=command, user=user, volumes=volumes, entrypoint=entrypoint)
+            return self.docker.run(image=image, command=command, user=user, volumes=volumes,
+                                   entrypoint=entrypoint, envs=env, detach=detach, publish_all=True)
         except DockerException as e:
             raise DeployerException(e)
 

stack_orchestrator/deploy/deployer.py

@@ -44,7 +44,7 @@ class Deployer(ABC):
         pass
 
     @abstractmethod
-    def run(self, image, command, user, volumes, entrypoint):
+    def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False):
         pass
 
 

stack_orchestrator/deploy/k8s/deploy_k8s.py

@@ -120,7 +120,7 @@ class K8sDeployer(Deployer):
         log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
         return log_stream_from_string(log_data)
 
-    def run(self, image, command, user, volumes, entrypoint=None):
+    def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False):
         # We need to figure out how to do this -- check why we're being called first
         pass
 

stack_orchestrator/deploy/run_webapp.py

@@ -0,0 +1,59 @@
+# Copyright © 2022, 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+# Builds webapp containers
+
+# env vars:
+# CERC_REPO_BASE_DIR defaults to ~/cerc
+
+# TODO: display the available list of containers; allow re-build of either all or specific containers
+
+import hashlib
+import click
+
+from dotenv import dotenv_values
+from stack_orchestrator.deploy.deployer_factory import getDeployer
+
+
+@click.command()
+@click.option("--image", help="image to deploy", required=True)
+@click.option("--deploy-to", default="compose", help="deployment type ([Docker] 'compose' or 'k8s')")
+@click.option("--env-file", help="environment file for webapp")
+@click.pass_context
+def command(ctx, image, deploy_to, env_file):
+    '''build the specified webapp container'''
+
+    env = {}
+    if env_file:
+        env = dotenv_values(env_file)
+
+    unique_cluster_descriptor = f"{image},{env}"
+    hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()
+    cluster = f"laconic-webapp-{hash}"
+
+    deployer = getDeployer(deploy_to,
+                           deployment_dir=None,
+                           compose_files=None,
+                           compose_project_name=cluster,
+                           compose_env_file=None)
+
+    container = deployer.run(image, command=[], user=None, volumes=[], entrypoint=None, env=env, detach=True)
+
+    # Make configurable?
+    webappPort = "3000/tcp"
+    # TODO: This assumes a Docker container object...
+    if webappPort in container.network_settings.ports:
+        mapping = container.network_settings.ports[webappPort][0]
+        print(f"""Image: {image}\nID: {container.id}\nURL: http://localhost:{mapping['HostPort']}""")

stack_orchestrator/main.py

@@ -20,6 +20,7 @@ from stack_orchestrator.repos import setup_repositories
 from stack_orchestrator.build import build_containers
 from stack_orchestrator.build import build_npms
 from stack_orchestrator.build import build_webapp
+from stack_orchestrator.deploy import run_webapp
 from stack_orchestrator.deploy import deploy
 from stack_orchestrator import version
 from stack_orchestrator.deploy import deployment
@@ -50,6 +51,7 @@ cli.add_command(setup_repositories.command, "setup-repositories")
 cli.add_command(build_containers.command, "build-containers")
 cli.add_command(build_npms.command, "build-npms")
 cli.add_command(build_webapp.command, "build-webapp")
+cli.add_command(run_webapp.command, "run-webapp")
 cli.add_command(deploy.command, "deploy")  # deploy is an alias for deploy-system
 cli.add_command(deploy.command, "deploy-system")
 cli.add_command(deployment.command, "deployment")

stack_orchestrator/repos/setup_repositories.py

@@ -25,7 +25,8 @@ import click
 import importlib.resources
 from pathlib import Path
 import yaml
-from stack_orchestrator.util import include_exclude_check
+from stack_orchestrator.constants import stack_file_name
+from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
 
 
 class GitProgress(git.RemoteProgress):
@@ -238,13 +239,20 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches
 
     repos_in_scope = []
     if stack:
-        # In order to be compatible with Python 3.8 we need to use this hack to get the path:
-        # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
-        stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, "stack.yml")
+        if stack_is_external(stack):
+            stack_file_path = Path(stack).joinpath(stack_file_name)
+        else:
+            # In order to be compatible with Python 3.8 we need to use this hack to get the path:
+            # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
+            stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name)
+        if not stack_file_path.exists():
+            error_exit(f"stack {stack} does not exist")
         with stack_file_path:
             stack_config = yaml.safe_load(open(stack_file_path, "r"))
-            # TODO: syntax check the input here
-            repos_in_scope = stack_config['repos']
+            if "repos" not in stack_config:
+                error_exit(f"stack {stack} does not define any repositories")
+            else:
+                repos_in_scope = stack_config["repos"]
     else:
         repos_in_scope = all_repos
 

stack_orchestrator/util.py

@@ -150,6 +150,12 @@ def get_parsed_deployment_spec(spec_file):
         sys.exit(1)
 
 
+def stack_is_external(stack: str):
+    # Bit of a hack: if the supplied stack string represents
+    # a path that exists then we assume it must be external
+    return Path(stack).exists() if stack is not None else False
+
+
 def get_yaml():
     # See: https://stackoverflow.com/a/45701840/1701505
     yaml = ruamel.yaml.YAML()
@@ -167,3 +173,8 @@ def global_options(ctx):
 # TODO: hack
 def global_options2(ctx):
     return ctx.parent.obj
+
+
+def error_exit(s):
+    print(f"ERROR: {s}")
+    sys.exit(1)

tests/webapp-test/run-webapp-test.sh

@@ -1,8 +1,10 @@
 #!/usr/bin/env bash
 set -e
+
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
+
 # Dump environment variables for debugging
 echo "Environment variables:"
 env
@@ -28,16 +30,18 @@ CHECK="SPECIAL_01234567890_TEST_STRING"
 
 set +e
 
-CONTAINER_ID=$(docker run -p 3000:3000 -d cerc/test-progressive-web-app:local)
+CONTAINER_ID=$(docker run -p 3000:3000 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local)
 sleep 3
-wget -O test.before -m http://localhost:3000
+wget -t 7 -O test.before -m http://localhost:3000
 
+docker logs $CONTAINER_ID
 docker remove -f $CONTAINER_ID
 
-CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$CHECK -d cerc/test-progressive-web-app:local)
+CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local)
 sleep 3
-wget -O test.after -m http://localhost:3000
+wget -t 7 -O test.after -m http://localhost:3000
 
+docker logs $CONTAINER_ID
 docker remove -f $CONTAINER_ID
 
 echo "###########################################################################"