Enable webapp test in GitHub CI. (#627)

* Add new build-webapp command and related scripts and containers.

.gitea/workflows/test-webapp.yml

@@ -0,0 +1,49 @@
+name: Webapp Test
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches:
+      - main
+      - ci-test
+    paths-ignore:
+      - '.gitea/workflows/triggers/*'
+
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
+jobs:
+  test:
+    name: "Run webapp test suite"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: Start dockerd # Also needed until we can incorporate into the executor
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
+      - name: "Run webapp tests"
+        run: ./tests/webapp-test/run-webapp-test.sh

.github/workflows/test-webapp.yml

@@ -0,0 +1,29 @@
+name: Webapp Test
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches: '*'
+
+jobs:
+  test:
+    name: "Run webapp test suite"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Run webapp tests"
+        run: ./tests/webapp-test/run-webapp-test.sh

.gitignore

@@ -6,5 +6,5 @@ laconic_stack_orchestrator.egg-info
 __pycache__
 *~
 package
-app/data/build_tag.txt
+stack_orchestrator/data/build_tag.txt
 /build

README.md

@@ -64,12 +64,12 @@ laconic-so update
 
 ## Usage
 
-The various [stacks](/app/data/stacks) each contain instructions for running different stacks based on your use case. For example:
+The various [stacks](/stack_orchestrator/data/stacks) each contain instructions for running different stacks based on your use case. For example:
 
-- [self-hosted Gitea](/app/data/stacks/build-support)
-- [an Optimism Fixturenet](/app/data/stacks/fixturenet-optimism)
-- [laconicd with console and CLI](app/data/stacks/fixturenet-laconic-loaded)
-- [kubo (IPFS)](app/data/stacks/kubo)
+- [self-hosted Gitea](/stack_orchestrator/data/stacks/build-support)
+- [an Optimism Fixturenet](/stack_orchestrator/data/stacks/fixturenet-optimism)
+- [laconicd with console and CLI](stack_orchestrator/data/stacks/fixturenet-laconic-loaded)
+- [kubo (IPFS)](stack_orchestrator/data/stacks/kubo)
 
 ## Contributing
 

app/deploy/k8s/helpers.py

@@ -1,57 +0,0 @@
-# Copyright © 2023 Vulcanize
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
-
-from kubernetes import client
-import subprocess
-from typing import Set
-
-from app.opts import opts
-
-
-def _run_command(command: str):
-    if opts.o.debug:
-        print(f"Running: {command}")
-    result = subprocess.run(command, shell=True)
-    if opts.o.debug:
-        print(f"Result: {result}")
-
-
-def create_cluster(name: str):
-    _run_command(f"kind create cluster --name {name}")
-
-
-def destroy_cluster(name: str):
-    _run_command(f"kind delete cluster --name {name}")
-
-
-def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]):
-    for image in image_set:
-        _run_command(f"kind load docker-image {image} --name {kind_cluster_name}")
-
-
-def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
-    pods = []
-    pod_response = core_api.list_namespaced_pod(namespace="default", label_selector="app=test-app")
-    if opts.o.debug:
-        print(f"pod_response: {pod_response}")
-    for pod_info in pod_response.items:
-        pod_name = pod_info.metadata.name
-        pods.append(pod_name)
-    return pods
-
-
-def log_stream_from_string(s: str):
-    # Note response has to be UTF-8 encoded because the caller expects to decode it
-    yield ("ignore", s.encode())

docs/adding-a-new-stack.md

@@ -8,7 +8,7 @@ Core to the feature completeness of stack orchestrator is to [decouple the tool
 
 ## Example
 
-- in `app/data/stacks/my-new-stack/stack.yml` add:
+- in `stack_orchestrator/data/stacks/my-new-stack/stack.yml` add:
 
 ```yaml
 version: "0.1"
@@ -21,7 +21,7 @@ pods:
   - my-new-stack
 ```
 
-- in `app/data/container-build/cerc-my-new-stack/build.sh` add:
+- in `stack_orchestrator/data/container-build/cerc-my-new-stack/build.sh` add:
 
 ```yaml
 #!/usr/bin/env bash
@@ -30,7 +30,7 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 docker build -t cerc/my-new-stack:local -f ${CERC_REPO_BASE_DIR}/my-new-stack/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/my-new-stack
 ```
 
-- in `app/data/compose/docker-compose-my-new-stack.yml` add:
+- in `stack_orchestrator/data/compose/docker-compose-my-new-stack.yml` add:
 
 ```yaml
 version: "3.2"
@@ -43,20 +43,20 @@ services:
       - "0.0.0.0:3000:3000"
 ```
 
-- in `app/data/repository-list.txt` add:
+- in `stack_orchestrator/data/repository-list.txt` add:
 
 ```bash
 github.com/my-org/my-new-stack
 ```
 whereby that repository contains your source code and a `Dockerfile`, and matches the `repos:` field in the `stack.yml`.
 
-- in `app/data/container-image-list.txt` add:
+- in `stack_orchestrator/data/container-image-list.txt` add:
 
 ```bash
 cerc/my-new-stack
 ```
 
-- in `app/data/pod-list.txt` add:
+- in `stack_orchestrator/data/pod-list.txt` add:
 
 ```bash
 my-new-stack

scripts/create_build_tag_file.sh

@@ -1,6 +1,6 @@
-build_tag_file_name=./app/data/build_tag.txt
+build_tag_file_name=./stack_orchestrator/data/build_tag.txt
 echo "# This file should be re-generated running: scripts/create_build_tag_file.sh script" > $build_tag_file_name
-product_version_string=$( tail -1 ./app/data/version.txt )
+product_version_string=$( tail -1 ./stack_orchestrator/data/version.txt )
 commit_string=$( git rev-parse --short HEAD )
 timestamp_string=$(date +'%Y%m%d%H%M')
 build_tag_string=${product_version_string}-${commit_string}-${timestamp_string}

setup.py

@@ -14,7 +14,7 @@ setup(
     long_description=long_description,
     long_description_content_type="text/markdown",
     url='https://github.com/cerc-io/stack-orchestrator',
-    py_modules=['cli', 'app'],
+    py_modules=['stack_orchestrator'],
     packages=find_packages(),
     install_requires=[requirements],
     python_requires='>=3.7',
@@ -25,6 +25,6 @@ setup(
         "Operating System :: OS Independent",
     ],
     entry_points={
-        'console_scripts': ['laconic-so=cli:cli'],
+        'console_scripts': ['laconic-so=stack_orchestrator.main:cli'],
     }
 )

stack_orchestrator/base.py

@@ -15,7 +15,7 @@
 
 import os
 from abc import ABC, abstractmethod
-from app.deploy.deploy import get_stack_status
+from stack_orchestrator.deploy.deploy import get_stack_status
 from decouple import config
 
 

stack_orchestrator/build/build_containers.py

@@ -27,12 +27,79 @@ import subprocess
 import click
 import importlib.resources
 from pathlib import Path
-from app.util import include_exclude_check, get_parsed_stack_config
-from app.base import get_npm_registry_url
+from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config
+from stack_orchestrator.base import get_npm_registry_url
 
 # TODO: find a place for this
 #    epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
 
+def make_container_build_env(dev_root_path: str,
+                             container_build_dir: str,
+                             debug: bool,
+                             force_rebuild: bool,
+                             extra_build_args: str):
+    container_build_env = {
+        "CERC_NPM_REGISTRY_URL": get_npm_registry_url(),
+        "CERC_GO_AUTH_TOKEN": config("CERC_GO_AUTH_TOKEN", default=""),
+        "CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default=""),
+        "CERC_REPO_BASE_DIR": dev_root_path,
+        "CERC_CONTAINER_BASE_DIR": container_build_dir,
+        "CERC_HOST_UID": f"{os.getuid()}",
+        "CERC_HOST_GID": f"{os.getgid()}",
+        "DOCKER_BUILDKIT": config("DOCKER_BUILDKIT", default="0")
+    }
+    container_build_env.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
+    container_build_env.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {})
+    container_build_env.update({"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {})
+    docker_host_env = os.getenv("DOCKER_HOST")
+    if docker_host_env:
+        container_build_env.update({"DOCKER_HOST": docker_host_env})
+
+    return container_build_env
+
+
+def process_container(container,
+                      container_build_dir: str,
+                      container_build_env: dict,
+                      dev_root_path: str,
+                      quiet: bool,
+                      verbose: bool,
+                      dry_run: bool,
+                      continue_on_error: bool,
+                      ):
+    if not quiet:
+        print(f"Building: {container}")
+    build_dir = os.path.join(container_build_dir, container.replace("/", "-"))
+    build_script_filename = os.path.join(build_dir, "build.sh")
+    if verbose:
+        print(f"Build script filename: {build_script_filename}")
+    if os.path.exists(build_script_filename):
+        build_command = build_script_filename
+    else:
+        if verbose:
+            print(f"No script file found: {build_script_filename}, using default build script")
+        repo_dir = container.split('/')[1]
+        # TODO: make this less of a hack -- should be specified in some metadata somewhere
+        # Check if we have a repo for this container. If not, set the context dir to the container-build subdir
+        repo_full_path = os.path.join(dev_root_path, repo_dir)
+        repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
+        build_command = os.path.join(container_build_dir,
+                                     "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}"
+    if not dry_run:
+        if verbose:
+            print(f"Executing: {build_command} with environment: {container_build_env}")
+        build_result = subprocess.run(build_command, shell=True, env=container_build_env)
+        if verbose:
+            print(f"Return code is: {build_result.returncode}")
+        if build_result.returncode != 0:
+            print(f"Error running build for {container}")
+            if not continue_on_error:
+                print("FATAL Error: container build failed and --continue-on-error not set, exiting")
+                sys.exit(1)
+            else:
+                print("****** Container Build Error, continuing because --continue-on-error is set")
+    else:
+        print("Skipped")
 
 @click.command()
 @click.option('--include', help="only build these containers")
@@ -67,7 +134,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
         print('Dev root directory doesn\'t exist, creating')
 
     # See: https://stackoverflow.com/a/20885799/1701505
-    from app import data
+    from stack_orchestrator import data
     with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
         all_containers = container_list_file.read().splitlines()
 
@@ -83,61 +150,16 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
         if stack:
             print(f"Stack: {stack}")
 
-    # TODO: make this configurable
-    container_build_env = {
-        "CERC_NPM_REGISTRY_URL": get_npm_registry_url(),
-        "CERC_GO_AUTH_TOKEN": config("CERC_GO_AUTH_TOKEN", default=""),
-        "CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default=""),
-        "CERC_REPO_BASE_DIR": dev_root_path,
-        "CERC_CONTAINER_BASE_DIR": container_build_dir,
-        "CERC_HOST_UID": f"{os.getuid()}",
-        "CERC_HOST_GID": f"{os.getgid()}",
-        "DOCKER_BUILDKIT": config("DOCKER_BUILDKIT", default="0")
-    }
-    container_build_env.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
-    container_build_env.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {})
-    container_build_env.update({"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {})
-    docker_host_env = os.getenv("DOCKER_HOST")
-    if docker_host_env:
-        container_build_env.update({"DOCKER_HOST": docker_host_env})
-
-    def process_container(container):
-        if not quiet:
-            print(f"Building: {container}")
-        build_dir = os.path.join(container_build_dir, container.replace("/", "-"))
-        build_script_filename = os.path.join(build_dir, "build.sh")
-        if verbose:
-            print(f"Build script filename: {build_script_filename}")
-        if os.path.exists(build_script_filename):
-            build_command = build_script_filename
-        else:
-            if verbose:
-                print(f"No script file found: {build_script_filename}, using default build script")
-            repo_dir = container.split('/')[1]
-            # TODO: make this less of a hack -- should be specified in some metadata somewhere
-            # Check if we have a repo for this container. If not, set the context dir to the container-build subdir
-            repo_full_path = os.path.join(dev_root_path, repo_dir)
-            repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
-            build_command = os.path.join(container_build_dir, "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}"
-        if not dry_run:
-            if verbose:
-                print(f"Executing: {build_command} with environment: {container_build_env}")
-            build_result = subprocess.run(build_command, shell=True, env=container_build_env)
-            if verbose:
-                print(f"Return code is: {build_result.returncode}")
-            if build_result.returncode != 0:
-                print(f"Error running build for {container}")
-                if not continue_on_error:
-                    print("FATAL Error: container build failed and --continue-on-error not set, exiting")
-                    sys.exit(1)
-                else:
-                    print("****** Container Build Error, continuing because --continue-on-error is set")
-        else:
-            print("Skipped")
+    container_build_env = make_container_build_env(dev_root_path,
+                                                   container_build_dir,
+                                                   debug,
+                                                   force_rebuild,
+                                                   extra_build_args)
 
     for container in containers_in_scope:
         if include_exclude_check(container, include, exclude):
-            process_container(container)
+            process_container(container, container_build_dir, container_build_env,
+                              dev_root_path, quiet, verbose, dry_run, continue_on_error)
         else:
             if verbose:
                 print(f"Excluding: {container}")

stack_orchestrator/build/build_npms.py

@@ -25,8 +25,8 @@ from decouple import config
 import click
 import importlib.resources
 from python_on_whales import docker, DockerException
-from app.base import get_stack
-from app.util import include_exclude_check, get_parsed_stack_config
+from stack_orchestrator.base import get_stack
+from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config
 
 builder_js_image_name = "cerc/builder-js:local"
 
@@ -83,7 +83,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
         os.makedirs(build_root_path)
 
     # See: https://stackoverflow.com/a/20885799/1701505
-    from app import data
+    from stack_orchestrator import data
     with importlib.resources.open_text(data, "npm-package-list.txt") as package_list_file:
         all_packages = package_list_file.read().splitlines()
 

stack_orchestrator/build/build_webapp.py

@@ -0,0 +1,76 @@
+# Copyright © 2022, 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+# Builds webapp containers
+
+# env vars:
+# CERC_REPO_BASE_DIR defaults to ~/cerc
+
+# TODO: display the available list of containers; allow re-build of either all or specific containers
+
+import os
+from decouple import config
+import click
+from pathlib import Path
+from stack_orchestrator.build import build_containers
+
+
+@click.command()
+@click.option('--base-container', default="cerc/nextjs-base")
+@click.option('--source-repo', help="directory containing the webapp to build", required=True)
+@click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
+@click.option("--extra-build-args", help="Supply extra arguments to build")
+@click.pass_context
+def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
+    '''build the specified webapp container'''
+
+    quiet = ctx.obj.quiet
+    verbose = ctx.obj.verbose
+    dry_run = ctx.obj.dry_run
+    debug = ctx.obj.debug
+    local_stack = ctx.obj.local_stack
+    stack = ctx.obj.stack
+    continue_on_error = ctx.obj.continue_on_error
+
+    # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
+    container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
+
+    if local_stack:
+        dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
+        print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
+    else:
+        dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
+
+    if not quiet:
+        print(f'Dev Root is: {dev_root_path}')
+
+    # First build the base container.
+    container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
+                                                                    force_rebuild, extra_build_args)
+
+    build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet,
+                                       verbose, dry_run, continue_on_error)
+
+
+    # Now build the target webapp.  We use the same build script, but with a different Dockerfile and work dir.
+    container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo)
+    container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir,
+                                                                    base_container.replace("/", "-"),
+                                                                    "Dockerfile.webapp")
+    webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
+    container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
+
+    build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet,
+                                       verbose, dry_run, continue_on_error)