Merge branch 'main' into ci-test

Change to trigger CI on Gitea

.gitea/workflows/fixturenet-eth-plugeth-arm-test.yml

@@ -1,45 +0,0 @@
-name: Fixturenet-Eth-Plugeth-Arm-Test
-
-on:
-  push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-arm-test'
-  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
-    - cron: '2 14 * * *'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
-
-jobs:
-  test:
-    name: "Run an Ethereum plugeth fixturenet test"
-    runs-on: ubuntu-latest-arm
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: "Run fixturenet-eth tests"
-        run: ./tests/fixturenet-eth-plugeth/run-test.sh

.gitea/workflows/test-database.yml

@@ -48,5 +48,5 @@ jobs:
         run: |
           source /opt/bash-utils/cgroup-helper.sh
           join_cgroup
-          ./tests/database/run-test.sh
+          CERC_SCRIPT_DEBUG=1 ./tests/database/run-test.sh
 

.gitea/workflows/triggers/fixturenet-eth-plugeth-arm-test

@@ -1,2 +0,0 @@
-Change this file to trigger running the fixturenet-eth-plugeth-arm-test CI job
-

stack_orchestrator/build/build_containers.py

@@ -27,11 +27,8 @@ import subprocess
 import click
 import importlib.resources
 from pathlib import Path
-from stack_orchestrator.opts import opts
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, error_exit, warn_exit
+from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, warn_exit
 from stack_orchestrator.base import get_npm_registry_url
-from stack_orchestrator.build.build_types import BuildContext
-from stack_orchestrator.build.publish import publish_image
 
 # TODO: find a place for this
 #    epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
@@ -62,58 +59,69 @@ def make_container_build_env(dev_root_path: str,
     return container_build_env
 
 
-def process_container(build_context: BuildContext) -> bool:
-    if not opts.o.quiet:
-        print(f"Building: {build_context.container}")
+def process_container(stack: str,
+                      container,
+                      container_build_dir: str,
+                      container_build_env: dict,
+                      dev_root_path: str,
+                      quiet: bool,
+                      verbose: bool,
+                      dry_run: bool,
+                      continue_on_error: bool,
+                      ):
+    if not quiet:
+        print(f"Building: {container}")
 
-    default_container_tag = f"{build_context.container}:local"
-    build_context.container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
+    default_container_tag = f"{container}:local"
+    container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
 
     # Check if this is in an external stack
-    if stack_is_external(build_context.stack):
-        container_parent_dir = Path(build_context.stack).joinpath("container-build")
-        temp_build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
+    if stack_is_external(stack):
+        container_parent_dir = Path(stack).joinpath("container-build")
+        temp_build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
         temp_build_script_filename = temp_build_dir.joinpath("build.sh")
         # Now check if the container exists in the external stack.
         if not temp_build_script_filename.exists():
             # If not, revert to building an internal container
-            container_parent_dir = build_context.container_build_dir
+            container_parent_dir = container_build_dir
     else:
-        container_parent_dir = build_context.container_build_dir
+        container_parent_dir = container_build_dir
 
-    build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
+    build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
     build_script_filename = build_dir.joinpath("build.sh")
 
-    if opts.o.verbose:
+    if verbose:
         print(f"Build script filename: {build_script_filename}")
     if os.path.exists(build_script_filename):
         build_command = build_script_filename.as_posix()
     else:
-        if opts.o.verbose:
+        if verbose:
             print(f"No script file found: {build_script_filename}, using default build script")
-        repo_dir = build_context.container.split('/')[1]
+        repo_dir = container.split('/')[1]
         # TODO: make this less of a hack -- should be specified in some metadata somewhere
         # Check if we have a repo for this container. If not, set the context dir to the container-build subdir
-        repo_full_path = os.path.join(build_context.dev_root_path, repo_dir)
+        repo_full_path = os.path.join(dev_root_path, repo_dir)
         repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
-        build_command = os.path.join(build_context.container_build_dir,
+        build_command = os.path.join(container_build_dir,
                                      "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
-    if not opts.o.dry_run:
+    if not dry_run:
         # No PATH at all causes failures with podman.
-        if "PATH" not in build_context.container_build_env:
-            build_context.container_build_env["PATH"] = os.environ["PATH"]
-        if opts.o.verbose:
-            print(f"Executing: {build_command} with environment: {build_context.container_build_env}")
-        build_result = subprocess.run(build_command, shell=True, env=build_context.container_build_env)
-        if opts.o.verbose:
+        if "PATH" not in container_build_env:
+            container_build_env["PATH"] = os.environ["PATH"]
+        if verbose:
+            print(f"Executing: {build_command} with environment: {container_build_env}")
+        build_result = subprocess.run(build_command, shell=True, env=container_build_env)
+        if verbose:
             print(f"Return code is: {build_result.returncode}")
         if build_result.returncode != 0:
-            return False
-        else:
-            return True
+            print(f"Error running build for {container}")
+            if not continue_on_error:
+                print("FATAL Error: container build failed and --continue-on-error not set, exiting")
+                sys.exit(1)
+            else:
+                print("****** Container Build Error, continuing because --continue-on-error is set")
     else:
         print("Skipped")
-        return True
 
 
 @click.command()
@@ -121,14 +129,17 @@ def process_container(build_context: BuildContext) -> bool:
 @click.option('--exclude', help="don\'t build these containers")
 @click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
 @click.option("--extra-build-args", help="Supply extra arguments to build")
-@click.option("--publish-images", is_flag=True, default=False, help="Publish the built images in the specified image registry")
-@click.option("--image-registry", help="Specify the image registry for --publish-images")
 @click.pass_context
-def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_images, image_registry):
+def command(ctx, include, exclude, force_rebuild, extra_build_args):
     '''build the set of containers required for a complete stack'''
 
+    quiet = ctx.obj.quiet
+    verbose = ctx.obj.verbose
+    dry_run = ctx.obj.dry_run
+    debug = ctx.obj.debug
     local_stack = ctx.obj.local_stack
     stack = ctx.obj.stack
+    continue_on_error = ctx.obj.continue_on_error
 
     # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
     container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
@@ -139,16 +150,12 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag
     else:
         dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
 
-    if not opts.o.quiet:
+    if not quiet:
         print(f'Dev Root is: {dev_root_path}')
 
     if not os.path.isdir(dev_root_path):
         print('Dev root directory doesn\'t exist, creating')
 
-    if publish_images:
-        if not image_registry:
-            error_exit("--image-registry must be supplied with --publish-images")
-
     # See: https://stackoverflow.com/a/20885799/1701505
     from stack_orchestrator import data
     with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
@@ -163,38 +170,21 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag
     else:
         containers_in_scope = all_containers
 
-    if opts.o.verbose:
+    if verbose:
         print(f'Containers: {containers_in_scope}')
         if stack:
             print(f"Stack: {stack}")
 
     container_build_env = make_container_build_env(dev_root_path,
                                                    container_build_dir,
-                                                   opts.o.debug,
+                                                   debug,
                                                    force_rebuild,
                                                    extra_build_args)
 
     for container in containers_in_scope:
         if include_exclude_check(container, include, exclude):
-
-            build_context = BuildContext(
-                stack,
-                container,
-                container_build_dir,
-                container_build_env,
-                dev_root_path
-            )
-            result = process_container(build_context)
-            if result:
-                if publish_images:
-                    publish_image(f"{container}:local", image_registry)
-            else:
-                print(f"Error running build for {build_context.container}")
-                if not opts.o.continue_on_error:
-                    error_exit("container build failed and --continue-on-error not set, exiting")
-                    sys.exit(1)
-                else:
-                    print("****** Container Build Error, continuing because --continue-on-error is set")
+            process_container(stack, container, container_build_dir, container_build_env,
+                              dev_root_path, quiet, verbose, dry_run, continue_on_error)
         else:
-            if opts.o.verbose:
+            if verbose:
                 print(f"Excluding: {container}")

stack_orchestrator/build/build_types.py

@@ -1,29 +0,0 @@
-# Copyright © 2024 Vulcanize
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
-
-
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Mapping
-
-
-@dataclass
-class BuildContext:
-    stack: str
-    container: str
-    container_build_dir: Path
-    container_build_env: Mapping[str,str]
-    dev_root_path: str
-

stack_orchestrator/build/build_webapp.py

@@ -21,14 +21,11 @@
 # TODO: display the available list of containers; allow re-build of either all or specific containers
 
 import os
-import sys
-
 from decouple import config
 import click
 from pathlib import Path
 from stack_orchestrator.build import build_containers
 from stack_orchestrator.deploy.webapp.util import determine_base_container
-from stack_orchestrator.build.build_types import BuildContext
 
 
 @click.command()
@@ -42,9 +39,12 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     '''build the specified webapp container'''
 
     quiet = ctx.obj.quiet
+    verbose = ctx.obj.verbose
+    dry_run = ctx.obj.dry_run
     debug = ctx.obj.debug
     local_stack = ctx.obj.local_stack
     stack = ctx.obj.stack
+    continue_on_error = ctx.obj.continue_on_error
 
     # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
     container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
@@ -65,17 +65,8 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
                                                                     force_rebuild, extra_build_args)
 
-    build_context_1 = BuildContext(
-        stack,
-        base_container,
-        container_build_dir,
-        container_build_env,
-        dev_root_path,
-    )
-    ok = build_containers.process_container(build_context_1)
-    if not ok:
-        print("ERROR: Build failed.", file=sys.stderr)
-        sys.exit(1)
+    build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
+                                       verbose, dry_run, continue_on_error)
 
     # Now build the target webapp.  We use the same build script, but with a different Dockerfile and work dir.
     container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
@@ -89,14 +80,5 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     else:
         container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag
 
-    build_context_2 = BuildContext(
-        stack,
-        base_container,
-        container_build_dir,
-        container_build_env,
-        dev_root_path,
-    )
-    ok = build_containers.process_container(build_context_2)
-    if not ok:
-        print("ERROR: Build failed.", file=sys.stderr)
-        sys.exit(1)
+    build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
+                                       verbose, dry_run, continue_on_error)

stack_orchestrator/build/publish.py

@@ -1,48 +0,0 @@
-# Copyright © 2024 Vulcanize
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
-
-from datetime import datetime
-from python_on_whales import DockerClient
-
-from stack_orchestrator.opts import opts
-from stack_orchestrator.util import error_exit
-
-
-def _publish_tag_for_image(local_image_tag: str, remote_repo: str, version: str):
-    # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
-    (image_name, image_version) = local_image_tag.split(":")
-    if image_version == "local":
-        return f"{remote_repo}/{image_name}:{version}"
-    else:
-        error_exit("Asked to publish a non-locally built image")
-
-
-def publish_image(local_tag, registry):
-    if opts.o.verbose:
-        print(f"Publishing this image: {local_tag} to this registry: {registry}")
-    docker = DockerClient()
-    # Figure out the target image tag
-    # Eventually this version will be generated from the source repo state
-    # Using a timestemp is an intermediate step
-    version = datetime.now().strftime("%Y%m%d%H%M")
-    remote_tag = _publish_tag_for_image(local_tag, registry, version)
-    # Tag the image thus
-    if opts.o.debug:
-        print(f"Tagging {local_tag} to {remote_tag}")
-    docker.image.tag(local_tag, remote_tag)
-    # Push it to the desired registry
-    if opts.o.verbose:
-        print(f"Pushing image {remote_tag}")
-    docker.image.push(remote_tag)

stack_orchestrator/data/compose/docker-compose-container-registry.yml

@@ -5,11 +5,9 @@ services:
     environment:
       REGISTRY_LOG_LEVEL: ${REGISTRY_LOG_LEVEL}
     volumes:
-      - config:/config:ro
       - registry-data:/var/lib/registry
     ports:
       - "5000"
 
 volumes:
-  config:
   registry-data:

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -26,8 +26,6 @@ RUN \
     && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
     # Install semver
     && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
-    # Install pnpm
-    && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
     && npm cache clean --force > /dev/null 2>&1
 
 # [Optional] Uncomment this section to install additional OS packages.
@@ -37,9 +35,6 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"
 
-# We do this to get a yq binary from the published container, for the correct architecture we're building here
-COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
-
 # Expose port for http
 EXPOSE 80
 

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh

@@ -10,12 +10,10 @@ TRG_DIR="${3:-.next-r}"
 
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
-  if [ -f "pnpm-lock.yaml" ]; then
-    CERC_BUILD_TOOL=pnpm
-  elif [ -f "yarn.lock" ]; then
-    CERC_BUILD_TOOL=yarn
-  else
+  if [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=npm
+  else
+    CERC_BUILD_TOOL=yarn
   fi
 fi
 

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -9,9 +9,7 @@ CERC_MIN_NEXTVER=13.4.2
 CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}"
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
-  if [ -f "pnpm-lock.yaml" ]; then
-    CERC_BUILD_TOOL=pnpm
-  elif [ -f "yarn.lock" ]; then
+  if [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
   else
     CERC_BUILD_TOOL=npm

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -16,9 +16,7 @@ trap ctrl_c INT
 
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
-  if [ -f "pnpm-lock.yaml" ]; then
-    CERC_BUILD_TOOL=pnpm
-  elif [ -f "yarn.lock" ]; then
+  if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then
     CERC_BUILD_TOOL=yarn
   else
     CERC_BUILD_TOOL=npm

stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile

@@ -24,10 +24,6 @@ RUN \
     && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
     # Install eslint
     && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
-    # Install semver
-    && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
-    # Install pnpm
-    && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
     && npm cache clean --force > /dev/null 2>&1
 
 # [Optional] Uncomment this section to install additional OS packages.

stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp

@@ -1,12 +1,11 @@
 FROM cerc/webapp-base:local as builder
 
 ARG CERC_BUILD_TOOL
-ARG CERC_BUILD_OUTPUT_DIR
 
 WORKDIR /app
 COPY . .
-RUN rm -rf node_modules build dist .next*
-RUN /scripts/build-app.sh /app /data
+RUN rm -rf node_modules build .next*
+RUN /scripts/build-app.sh /app build /data
 
 FROM cerc/webapp-base:local
 COPY --from=builder /data /data

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh

@@ -7,10 +7,9 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
-CERC_BUILD_OUTPUT_DIR="${CERC_BUILD_OUTPUT_DIR}"
-
 WORK_DIR="${1:-/app}"
-DEST_DIR="${2:-/data}"
+OUTPUT_DIR="${2:-build}"
+DEST_DIR="${3:-/data}"
 
 if [ -f "${WORK_DIR}/build-webapp.sh" ]; then
   echo "Building webapp with ${WORK_DIR}/build-webapp.sh ..."
@@ -23,9 +22,7 @@ elif [ -f "${WORK_DIR}/package.json" ]; then
   cd "${WORK_DIR}" || exit 1
 
   if [ -z "$CERC_BUILD_TOOL" ]; then
-    if [ -f "pnpm-lock.yaml" ]; then
-      CERC_BUILD_TOOL=pnpm
-    elif [ -f "yarn.lock" ]; then
+    if [ -f "yarn.lock" ]; then
       CERC_BUILD_TOOL=yarn
     else
       CERC_BUILD_TOOL=npm
@@ -36,17 +33,7 @@ elif [ -f "${WORK_DIR}/package.json" ]; then
   $CERC_BUILD_TOOL build || exit 1
 
   rm -rf "${DEST_DIR}"
-  if [ -z "${CERC_BUILD_OUTPUT_DIR}" ]; then
-    if [ -d "${WORK_DIR}/dist" ]; then
-      CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/dist"
-    elif [ -d "${WORK_DIR}/build" ]; then
-      CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/build"
-    else
-      echo "ERROR: Unable to locate build output.  Set with --extra-build-args \"--build-arg CERC_BUILD_OUTPUT_DIR=path\"" 1>&2
-      exit 1
-    fi
-  fi
-  mv "${CERC_BUILD_OUTPUT_DIR}" "${DEST_DIR}"
+  mv "${WORK_DIR}/${OUTPUT_DIR}" "${DEST_DIR}"
 else
   echo "Copying static app ..."
   mv "${WORK_DIR}" "${DEST_DIR}"

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh

@@ -3,28 +3,13 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
-CERC_LISTEN_PORT=${CERC_LISTEN_PORT:-80}
 CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}"
 CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}"
-CERC_SINGLE_PAGE_APP="${CERC_SINGLE_PAGE_APP}"
-
-if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then
-  if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ] && [ -d "${CERC_WEBAPP_FILES_DIR}/static" ]; then
-    CERC_SINGLE_PAGE_APP=true
-  else
-    CERC_SINGLE_PAGE_APP=false
-  fi
-fi
 
 if [ "true" == "$CERC_ENABLE_CORS" ]; then
   CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --cors"
 fi
 
-if [ "true" == "$CERC_SINGLE_PAGE_APP" ]; then
-  # Create a catchall redirect back to /
-  CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --proxy http://localhost:${CERC_LISTEN_PORT}?"
-fi
-
 LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
 if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
   if [ -f "/config/laconic-hosted-config.yml" ]; then
@@ -35,8 +20,8 @@ if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
 fi
 
 if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
-  /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "${CERC_WEBAPP_FILES_DIR}"
+  /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE ${CERC_WEBAPP_FILES_DIR}
 fi
 
 /scripts/apply-runtime-env.sh ${CERC_WEBAPP_FILES_DIR}
-http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT} "${CERC_WEBAPP_FILES_DIR}"
+http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT:-80} ${CERC_WEBAPP_FILES_DIR}

stack_orchestrator/data/stacks/uniswap-urbit-app/README.md

@@ -33,29 +33,23 @@ laconic-so --stack uniswap-urbit-app deploy init --output uniswap-urbit-app-spec
 
 ### Ports
 
-Edit `uniswap-urbit-app-spec.yml` such that it looks like:
+Edit `network` in spec file to map container ports to same ports in host:
 
 ```yml
-stack: uniswap-urbit-app
-deploy-to: compose
+...
 network:
   ports:
-    proxy-server:
-      - '4000:4000'
     urbit-fake-ship:
       - '8080:80'
+    proxy-server:
+      - '4000:4000'
     ipfs:
-     - '4001'
-     - '8081:8080'
-     - 0.0.0.0:5001:5001
-volumes:
-  urbit_app_builds: ./data/urbit_app_builds
-  urbit_data: ./data/urbit_data
-  ipfs-import: ./data/ipfs-import
-  ipfs-data: ./data/ipfs-data
+      - '8081:8080'
+      - '5001:5001'
+...
 ```
 
-Note: Skip the `ipfs` ports if using an externally running IPFS node, set via `config.env`, below.
+Note: Skip the `ipfs` ports if need to use an externally running IPFS node
 
 ### Data volumes
 

stack_orchestrator/deploy/deployment_create.py

@@ -327,14 +327,12 @@ def init_operation(deploy_command_context, stack, deployer_type, config,
     default_spec_file_content = call_stack_deploy_init(deploy_command_context)
     spec_file_content = {"stack": stack, constants.deploy_to_key: deployer_type}
     if deployer_type == "k8s":
-        if kube_config:
-            spec_file_content.update({constants.kube_config_key: kube_config})
-        else:
+        if kube_config is None:
             error_exit("--kube-config must be supplied with --deploy-to k8s")
-        if image_registry:
-            spec_file_content.update({constants.image_registry_key: image_registry})
-        else:
-            print("WARNING: --image-registry not specified, only default container registries (eg, Docker Hub) will be available")
+        if image_registry is None:
+            error_exit("--image-registry must be supplied with --deploy-to k8s")
+        spec_file_content.update({constants.kube_config_key: kube_config})
+        spec_file_content.update({constants.image_registry_key: image_registry})
     else:
         # Check for --kube-config supplied for non-relevant deployer types
         if kube_config is not None:

stack_orchestrator/deploy/images.py

@@ -29,29 +29,6 @@ def _image_needs_pushed(image: str):
     return image.endswith(":local")
 
 
-def remote_image_exists(remote_repo_url: str, local_tag: str):
-    docker = DockerClient()
-    try:
-        remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
-        result = docker.manifest.inspect(remote_tag)
-        return True if result else False
-    except Exception:  # noqa: E722
-        return False
-
-
-def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
-    if not additional_tags:
-        return
-
-    if not remote_image_exists(remote_repo_url, local_tag):
-        raise Exception(f"{local_tag} does not exist in {remote_repo_url}")
-
-    docker = DockerClient()
-    remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
-    new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
-    docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags)
-
-
 def remote_tag_for_image(image: str, remote_repo_url: str):
     # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
     major_parts = image.split("/", 2)

stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py

@@ -24,7 +24,6 @@ import uuid
 
 import click
 
-from stack_orchestrator.deploy.images import remote_image_exists, add_tags_to_image
 from stack_orchestrator.deploy.webapp import deploy_webapp
 from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
                                                    build_container_image, push_container_image,
@@ -44,7 +43,6 @@ def process_app_deployment_request(
     deployment_parent_dir,
     kube_config,
     image_registry,
-    force_rebuild=False,
     log_file=None
 ):
     # 1. look up application
@@ -93,9 +91,7 @@ def process_app_deployment_request(
     deployment_record = laconic.get_record(app_deployment_crn)
     deployment_dir = os.path.join(deployment_parent_dir, fqdn)
     deployment_config_file = os.path.join(deployment_dir, "config.env")
-    # TODO: Is there any reason not to simplify the hash input to the app_deployment_crn?
     deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest()
-    app_image_shared_tag = f"laconic-webapp/{app.id}:local"
     #   b. check for deployment directory (create if necessary)
     if not os.path.exists(deployment_dir):
         if deployment_record:
@@ -110,20 +106,11 @@ def process_app_deployment_request(
     needs_k8s_deploy = False
     # 6. build container (if needed)
     if not deployment_record or deployment_record.attributes.application != app.id:
+        # TODO: pull from request
+        extra_build_args = []
+        build_container_image(app, deployment_container_tag, extra_build_args, log_file)
+        push_container_image(deployment_dir, log_file)
         needs_k8s_deploy = True
-        # check if the image already exists
-        shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag)
-        if shared_tag_exists and not force_rebuild:
-            # simply add our unique tag to the existing image and we are done
-            print(f"Using existing app image {app_image_shared_tag} for {deployment_container_tag}", file=log_file)
-            add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
-        else:
-            extra_build_args = []  # TODO: pull from request
-            build_container_image(app, deployment_container_tag, extra_build_args, log_file)
-            push_container_image(deployment_dir, log_file)
-            # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
-            print(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_tag}", file=log_file)
-            add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
 
     # 7. update config (if needed)
     if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config:
@@ -184,13 +171,12 @@ def dump_known_requests(filename, requests, status="SEEN"):
 @click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
 @click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="")
 @click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="")
-@click.option("--force-rebuild", help="Rebuild even if the image already exists.", is_flag=True)
 @click.option("--log-dir", help="Output build/deployment logs to directory.", default=None)
 @click.pass_context
 def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir,  # noqa: C901
             request_id, discover, state_file, only_update_state,
             dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run,
-            include_tags, exclude_tags, force_rebuild, log_dir):
+            include_tags, exclude_tags, log_dir):
     if request_id and discover:
         print("Cannot specify both --request-id and --discover", file=sys.stderr)
         sys.exit(2)
@@ -320,7 +306,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
                     os.path.abspath(deployment_parent_dir),
                     kube_config,
                     image_registry,
-                    force_rebuild,
                     run_log_file
                 )
                 status = "DEPLOYED"