diff --git a/stack_orchestrator/build/build_containers.py b/stack_orchestrator/build/build_containers.py
index 2c06afcf..71debf09 100644
--- a/stack_orchestrator/build/build_containers.py
+++ b/stack_orchestrator/build/build_containers.py
@@ -25,13 +25,13 @@ import sys
from decouple import config
import subprocess
import click
-import importlib.resources
from pathlib import Path
from stack_orchestrator.opts import opts
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, error_exit, warn_exit
+from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
from stack_orchestrator.base import get_npm_registry_url
from stack_orchestrator.build.build_types import BuildContext
from stack_orchestrator.build.publish import publish_image
+from stack_orchestrator.build.build_util import get_containers_in_scope
# TODO: find a place for this
# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
@@ -149,24 +149,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag
if not image_registry:
error_exit("--image-registry must be supplied with --publish-images")
- # See: https://stackoverflow.com/a/20885799/1701505
- from stack_orchestrator import data
- with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
- all_containers = container_list_file.read().splitlines()
-
- containers_in_scope = []
- if stack:
- stack_config = get_parsed_stack_config(stack)
- if "containers" not in stack_config or stack_config["containers"] is None:
- warn_exit(f"stack {stack} does not define any containers")
- containers_in_scope = stack_config['containers']
- else:
- containers_in_scope = all_containers
-
- if opts.o.verbose:
- print(f'Containers: {containers_in_scope}')
- if stack:
- print(f"Stack: {stack}")
+ containers_in_scope = get_containers_in_scope(stack)
container_build_env = make_container_build_env(dev_root_path,
container_build_dir,
diff --git a/stack_orchestrator/build/build_util.py b/stack_orchestrator/build/build_util.py
new file mode 100644
index 00000000..7eb89ba9
--- /dev/null
+++ b/stack_orchestrator/build/build_util.py
@@ -0,0 +1,43 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+import importlib.resources
+
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import get_parsed_stack_config, warn_exit
+
+
+def get_containers_in_scope(stack: str):
+
+ # See: https://stackoverflow.com/a/20885799/1701505
+ from stack_orchestrator import data
+ with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
+ all_containers = container_list_file.read().splitlines()
+
+ containers_in_scope = []
+ if stack:
+ stack_config = get_parsed_stack_config(stack)
+ if "containers" not in stack_config or stack_config["containers"] is None:
+ warn_exit(f"stack {stack} does not define any containers")
+ containers_in_scope = stack_config['containers']
+ else:
+ containers_in_scope = all_containers
+
+ if opts.o.verbose:
+ print(f'Containers: {containers_in_scope}')
+ if stack:
+ print(f"Stack: {stack}")
+
+ return containers_in_scope
\ No newline at end of file
diff --git a/stack_orchestrator/build/build_webapp.py b/stack_orchestrator/build/build_webapp.py
index fcdd8f66..25fbc850 100644
--- a/stack_orchestrator/build/build_webapp.py
+++ b/stack_orchestrator/build/build_webapp.py
@@ -21,6 +21,8 @@
# TODO: display the available list of containers; allow re-build of either all or specific containers
import os
+import sys
+
from decouple import config
import click
from pathlib import Path
@@ -40,12 +42,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
'''build the specified webapp container'''
quiet = ctx.obj.quiet
- verbose = ctx.obj.verbose
- dry_run = ctx.obj.dry_run
debug = ctx.obj.debug
local_stack = ctx.obj.local_stack
stack = ctx.obj.stack
- continue_on_error = ctx.obj.continue_on_error
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
@@ -73,7 +72,10 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
container_build_env,
dev_root_path,
)
- build_containers.process_container(build_context_1)
+ ok = build_containers.process_container(build_context_1)
+ if not ok:
+ print("ERROR: Build failed.", file=sys.stderr)
+ sys.exit(1)
# Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir.
container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
@@ -94,4 +96,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
container_build_env,
dev_root_path,
)
- build_containers.process_container(build_context_2)
+ ok = build_containers.process_container(build_context_2)
+ if not ok:
+ print("ERROR: Build failed.", file=sys.stderr)
+ sys.exit(1)
\ No newline at end of file
diff --git a/stack_orchestrator/build/fetch_containers.py b/stack_orchestrator/build/fetch_containers.py
new file mode 100644
index 00000000..890cb94f
--- /dev/null
+++ b/stack_orchestrator/build/fetch_containers.py
@@ -0,0 +1,180 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+import click
+from dataclasses import dataclass
+import json
+import platform
+from python_on_whales import DockerClient
+from python_on_whales.components.manifest.cli_wrapper import ManifestCLI, ManifestList
+from python_on_whales.utils import run
+import requests
+from typing import List
+
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import include_exclude_check, error_exit
+from stack_orchestrator.build.build_util import get_containers_in_scope
+
+# Experimental fetch-container command
+
+
+@dataclass
+class RegistryInfo:
+ registry: str
+ registry_username: str
+ registry_token: str
+
+
+# Extending this code to support the --verbose option, cnosider contributing upstream
+# https://github.com/gabrieldemarmiesse/python-on-whales/blob/master/python_on_whales/components/manifest/cli_wrapper.py#L129
+class ExtendedManifestCLI(ManifestCLI):
+ def inspect_verbose(self, x: str) -> ManifestList:
+ """Returns a Docker manifest list object."""
+ json_str = run(self.docker_cmd + ["manifest", "inspect", "--verbose", x])
+ return json.loads(json_str)
+
+
+def _local_tag_for(container: str):
+ return f"{container}:local"
+
+
+# See: https://docker-docs.uclv.cu/registry/spec/api/
+# Emulate this:
+# $ curl -u "my-username:my-token" -X GET "https:///v2/cerc-io/cerc/test-container/tags/list"
+# {"name":"cerc-io/cerc/test-container","tags":["202402232130","202402232208"]}
+def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List[str]:
+ # registry looks like: git.vdb.to/cerc-io
+ registry_parts = registry_info.registry.split("/")
+ url = f"https://{registry_parts[0]}/v2/{registry_parts[1]}/{container}/tags/list"
+ if opts.o.debug:
+ print(f"Fetching tags from: {url}")
+ response = requests.get(url, auth=(registry_info.registry_username, registry_info.registry_token))
+ if response.status_code == 200:
+ tag_info = response.json()
+ if opts.o.debug:
+ print(f"container tags list: {tag_info}")
+ tags_array = tag_info["tags"]
+ return tags_array
+ else:
+ error_exit(f"failed to fetch tags from image registry, status code: {response.status_code}")
+
+
+def _find_latest(candidate_tags: List[str]):
+ # Lex sort should give us the latest first
+ sorted_candidates = sorted(candidate_tags)
+ if opts.o.debug:
+ print(f"sorted candidates: {sorted_candidates}")
+ return sorted_candidates[0]
+
+
+def _filter_for_platform(container: str,
+ registry_info: RegistryInfo,
+ tag_list: List[str]) -> List[str] :
+ filtered_tags = []
+ this_machine = platform.machine()
+ # Translate between Python platform names and docker
+ if this_machine == "x86_64":
+ this_machine = "amd64"
+ if opts.o.debug:
+ print(f"Python says the architecture is: {this_machine}")
+ docker = DockerClient()
+ docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token)
+ for tag in tag_list:
+ remote_tag = f"{registry_info.registry}/{container}:{tag}"
+ manifest_cmd = ExtendedManifestCLI(docker.client_config)
+ manifest = manifest_cmd.inspect_verbose(remote_tag)
+ if opts.o.debug:
+ print(f"manifest: {manifest}")
+ image_architecture = manifest["Descriptor"]["platform"]["architecture"]
+ if opts.o.debug:
+ print(f"image_architecture: {image_architecture}")
+ if this_machine == image_architecture:
+ filtered_tags.append(tag)
+ if opts.o.debug:
+ print(f"Tags filtered for platform: {filtered_tags}")
+ return filtered_tags
+
+
+def _get_latest_image(container: str, registry_info: RegistryInfo):
+ all_tags = _get_tags_for_container(container, registry_info)
+ tags_for_platform = _filter_for_platform(container, registry_info, all_tags)
+ latest_tag = _find_latest(tags_for_platform)
+ return f"{container}:{latest_tag}"
+
+
+def _fetch_image(tag: str, registry_info: RegistryInfo):
+ docker = DockerClient()
+ docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token)
+ remote_tag = f"{registry_info.registry}/{tag}"
+ if opts.o.debug:
+ print(f"Attempting to pull this image: {remote_tag}")
+ docker.image.pull(remote_tag)
+
+
+def _exists_locally(container: str):
+ docker = DockerClient()
+ return docker.image.exists(_local_tag_for(container))
+
+
+def _add_local_tag(remote_tag: str, registry: str, local_tag: str):
+ docker = DockerClient()
+ docker.image.tag(f"{registry}/{remote_tag}", local_tag)
+
+
+@click.command()
+@click.option('--include', help="only fetch these containers")
+@click.option('--exclude', help="don\'t fetch these containers")
+@click.option("--force-local-overwrite", is_flag=True, default=False, help="Overwrite a locally built image, if present")
+@click.option("--image-registry", required=True, help="Specify the image registry to fetch from")
+@click.option("--registry-username", required=True, help="Specify the image registry username")
+@click.option("--registry-token", required=True, help="Specify the image registry access token")
+@click.pass_context
+def command(ctx, include, exclude, force_local_overwrite, image_registry, registry_username, registry_token):
+ '''EXPERIMENTAL: fetch the images for a stack from remote registry'''
+
+ registry_info = RegistryInfo(image_registry, registry_username, registry_token)
+ # Generate list of target containers
+ stack = ctx.obj.stack
+ containers_in_scope = get_containers_in_scope(stack)
+ for container in containers_in_scope:
+ local_tag = _local_tag_for(container)
+ if include_exclude_check(container, include, exclude):
+ if opts.o.debug:
+ print(f"Processing: {container}")
+ # For each container, attempt to find the latest of a set of
+ # images with the correct name and platform in the specified registry
+ image_to_fetch = _get_latest_image(container, registry_info)
+ if opts.o.debug:
+ print(f"Fetching: {image_to_fetch}")
+ _fetch_image(image_to_fetch, registry_info)
+ # Now check if the target container already exists exists locally already
+ if (_exists_locally(container)):
+ if not opts.o.quiet:
+ print(f"Container image {container} already exists locally")
+ # if so, fail unless the user specified force-local-overwrite
+ if (force_local_overwrite):
+ # In that case remove the existing :local tag
+ if not opts.o.quiet:
+ print(f"Warning: overwriting local tag from this image: {container} because "
+ "--force-local-overwrite was specified")
+ else:
+ if not opts.o.quiet:
+ print(f"Skipping local tagging for this image: {container} because that would "
+ "overwrite an existing :local tagged image, use --force-local-overwrite to do so.")
+ # Tag the fetched image with the :local tag
+ _add_local_tag(image_to_fetch, image_registry, local_tag)
+ else:
+ if opts.o.verbose:
+ print(f"Excluding: {container}")
diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile
index 5f9548ee..d3ae8237 100644
--- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile
+++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile
@@ -26,6 +26,8 @@ RUN \
&& su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
# Install semver
&& su ${USERNAME} -c "umask 0002 && npm install -g semver" \
+ # Install pnpm
+ && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
&& npm cache clean --force > /dev/null 2>&1
# [Optional] Uncomment this section to install additional OS packages.
@@ -35,6 +37,9 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
# [Optional] Uncomment if you want to install more global node modules
# RUN su node -c "npm install -g "
+# We do this to get a yq binary from the published container, for the correct architecture we're building here
+COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
+
# Expose port for http
EXPOSE 80
diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh
index a662ae02..ba0f725f 100755
--- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh
+++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh
@@ -10,10 +10,12 @@ TRG_DIR="${3:-.next-r}"
CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
if [ -z "$CERC_BUILD_TOOL" ]; then
- if [ -f "yarn.lock" ]; then
- CERC_BUILD_TOOL=npm
- else
+ if [ -f "pnpm-lock.yaml" ]; then
+ CERC_BUILD_TOOL=pnpm
+ elif [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=yarn
+ else
+ CERC_BUILD_TOOL=npm
fi
fi
diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh
index 40bcbeed..bf86265d 100755
--- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh
+++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh
@@ -9,7 +9,9 @@ CERC_MIN_NEXTVER=13.4.2
CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}"
CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
if [ -z "$CERC_BUILD_TOOL" ]; then
- if [ -f "yarn.lock" ]; then
+ if [ -f "pnpm-lock.yaml" ]; then
+ CERC_BUILD_TOOL=pnpm
+ elif [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=yarn
else
CERC_BUILD_TOOL=npm
diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh
index bd254572..4b69d935 100755
--- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh
+++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh
@@ -16,7 +16,9 @@ trap ctrl_c INT
CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
if [ -z "$CERC_BUILD_TOOL" ]; then
- if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then
+ if [ -f "pnpm-lock.yaml" ]; then
+ CERC_BUILD_TOOL=pnpm
+ elif [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=yarn
else
CERC_BUILD_TOOL=npm
diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile
index 9ae166b0..b8319c51 100644
--- a/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile
+++ b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile
@@ -24,6 +24,10 @@ RUN \
&& su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
# Install eslint
&& su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
+ # Install semver
+ && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
+ # Install pnpm
+ && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
&& npm cache clean --force > /dev/null 2>&1
# [Optional] Uncomment this section to install additional OS packages.
diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp
index 711eff25..9eaf46e1 100644
--- a/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp
+++ b/stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp
@@ -1,11 +1,12 @@
FROM cerc/webapp-base:local as builder
ARG CERC_BUILD_TOOL
+ARG CERC_BUILD_OUTPUT_DIR
WORKDIR /app
COPY . .
-RUN rm -rf node_modules build .next*
-RUN /scripts/build-app.sh /app build /data
+RUN rm -rf node_modules build dist .next*
+RUN /scripts/build-app.sh /app /data
FROM cerc/webapp-base:local
COPY --from=builder /data /data
diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh
index b515a142..3a63df84 100755
--- a/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh
+++ b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh
@@ -7,9 +7,10 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
fi
CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+CERC_BUILD_OUTPUT_DIR="${CERC_BUILD_OUTPUT_DIR}"
+
WORK_DIR="${1:-/app}"
-OUTPUT_DIR="${2:-build}"
-DEST_DIR="${3:-/data}"
+DEST_DIR="${2:-/data}"
if [ -f "${WORK_DIR}/build-webapp.sh" ]; then
echo "Building webapp with ${WORK_DIR}/build-webapp.sh ..."
@@ -22,7 +23,9 @@ elif [ -f "${WORK_DIR}/package.json" ]; then
cd "${WORK_DIR}" || exit 1
if [ -z "$CERC_BUILD_TOOL" ]; then
- if [ -f "yarn.lock" ]; then
+ if [ -f "pnpm-lock.yaml" ]; then
+ CERC_BUILD_TOOL=pnpm
+ elif [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=yarn
else
CERC_BUILD_TOOL=npm
@@ -33,7 +36,17 @@ elif [ -f "${WORK_DIR}/package.json" ]; then
$CERC_BUILD_TOOL build || exit 1
rm -rf "${DEST_DIR}"
- mv "${WORK_DIR}/${OUTPUT_DIR}" "${DEST_DIR}"
+ if [ -z "${CERC_BUILD_OUTPUT_DIR}" ]; then
+ if [ -d "${WORK_DIR}/dist" ]; then
+ CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/dist"
+ elif [ -d "${WORK_DIR}/build" ]; then
+ CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/build"
+ else
+ echo "ERROR: Unable to locate build output. Set with --extra-build-args \"--build-arg CERC_BUILD_OUTPUT_DIR=path\"" 1>&2
+ exit 1
+ fi
+ fi
+ mv "${CERC_BUILD_OUTPUT_DIR}" "${DEST_DIR}"
else
echo "Copying static app ..."
mv "${WORK_DIR}" "${DEST_DIR}"
diff --git a/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh
index f9aa2c33..4fa1dc03 100755
--- a/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh
+++ b/stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh
@@ -3,13 +3,28 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
+CERC_LISTEN_PORT=${CERC_LISTEN_PORT:-80}
CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}"
CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}"
+CERC_SINGLE_PAGE_APP="${CERC_SINGLE_PAGE_APP}"
+
+if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then
+ if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ] && [ -d "${CERC_WEBAPP_FILES_DIR}/static" ]; then
+ CERC_SINGLE_PAGE_APP=true
+ else
+ CERC_SINGLE_PAGE_APP=false
+ fi
+fi
if [ "true" == "$CERC_ENABLE_CORS" ]; then
CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --cors"
fi
+if [ "true" == "$CERC_SINGLE_PAGE_APP" ]; then
+ # Create a catchall redirect back to /
+ CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --proxy http://localhost:${CERC_LISTEN_PORT}?"
+fi
+
LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
if [ -f "/config/laconic-hosted-config.yml" ]; then
@@ -20,8 +35,8 @@ if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
fi
if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
- /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE ${CERC_WEBAPP_FILES_DIR}
+ /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "${CERC_WEBAPP_FILES_DIR}"
fi
/scripts/apply-runtime-env.sh ${CERC_WEBAPP_FILES_DIR}
-http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT:-80} ${CERC_WEBAPP_FILES_DIR}
+http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT} "${CERC_WEBAPP_FILES_DIR}"
\ No newline at end of file
diff --git a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md
index cdbb7f6a..187d723f 100644
--- a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md
+++ b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md
@@ -33,23 +33,29 @@ laconic-so --stack uniswap-urbit-app deploy init --output uniswap-urbit-app-spec
### Ports
-Edit `network` in spec file to map container ports to same ports in host:
+Edit `uniswap-urbit-app-spec.yml` such that it looks like:
```yml
-...
+stack: uniswap-urbit-app
+deploy-to: compose
network:
ports:
- urbit-fake-ship:
- - '8080:80'
proxy-server:
- '4000:4000'
+ urbit-fake-ship:
+ - '8080:80'
ipfs:
- - '8081:8080'
- - '5001:5001'
-...
+ - '4001'
+ - '8081:8080'
+ - 0.0.0.0:5001:5001
+volumes:
+ urbit_app_builds: ./data/urbit_app_builds
+ urbit_data: ./data/urbit_data
+ ipfs-import: ./data/ipfs-import
+ ipfs-data: ./data/ipfs-data
```
-Note: Skip the `ipfs` ports if need to use an externally running IPFS node
+Note: Skip the `ipfs` ports if using an externally running IPFS node, set via `config.env`, below.
### Data volumes
diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py
index 7ddcca33..77713d18 100644
--- a/stack_orchestrator/deploy/images.py
+++ b/stack_orchestrator/deploy/images.py
@@ -29,6 +29,29 @@ def _image_needs_pushed(image: str):
return image.endswith(":local")
+def remote_image_exists(remote_repo_url: str, local_tag: str):
+ docker = DockerClient()
+ try:
+ remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
+ result = docker.manifest.inspect(remote_tag)
+ return True if result else False
+ except Exception: # noqa: E722
+ return False
+
+
+def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
+ if not additional_tags:
+ return
+
+ if not remote_image_exists(remote_repo_url, local_tag):
+ raise Exception(f"{local_tag} does not exist in {remote_repo_url}")
+
+ docker = DockerClient()
+ remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
+ new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
+ docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags)
+
+
def remote_tag_for_image(image: str, remote_repo_url: str):
# Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
major_parts = image.split("/", 2)
diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py
index 15e89d9e..027a76b3 100644
--- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py
+++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py
@@ -24,6 +24,7 @@ import uuid
import click
+from stack_orchestrator.deploy.images import remote_image_exists, add_tags_to_image
from stack_orchestrator.deploy.webapp import deploy_webapp
from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
build_container_image, push_container_image,
@@ -43,6 +44,7 @@ def process_app_deployment_request(
deployment_parent_dir,
kube_config,
image_registry,
+ force_rebuild=False,
log_file=None
):
# 1. look up application
@@ -91,7 +93,9 @@ def process_app_deployment_request(
deployment_record = laconic.get_record(app_deployment_crn)
deployment_dir = os.path.join(deployment_parent_dir, fqdn)
deployment_config_file = os.path.join(deployment_dir, "config.env")
+ # TODO: Is there any reason not to simplify the hash input to the app_deployment_crn?
deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest()
+ app_image_shared_tag = f"laconic-webapp/{app.id}:local"
# b. check for deployment directory (create if necessary)
if not os.path.exists(deployment_dir):
if deployment_record:
@@ -106,11 +110,20 @@ def process_app_deployment_request(
needs_k8s_deploy = False
# 6. build container (if needed)
if not deployment_record or deployment_record.attributes.application != app.id:
- # TODO: pull from request
- extra_build_args = []
- build_container_image(app, deployment_container_tag, extra_build_args, log_file)
- push_container_image(deployment_dir, log_file)
needs_k8s_deploy = True
+ # check if the image already exists
+ shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag)
+ if shared_tag_exists and not force_rebuild:
+ # simply add our unique tag to the existing image and we are done
+ print(f"Using existing app image {app_image_shared_tag} for {deployment_container_tag}", file=log_file)
+ add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
+ else:
+ extra_build_args = [] # TODO: pull from request
+ build_container_image(app, deployment_container_tag, extra_build_args, log_file)
+ push_container_image(deployment_dir, log_file)
+ # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
+ print(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_tag}", file=log_file)
+ add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
# 7. update config (if needed)
if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config:
@@ -171,12 +184,13 @@ def dump_known_requests(filename, requests, status="SEEN"):
@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
@click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="")
@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="")
+@click.option("--force-rebuild", help="Rebuild even if the image already exists.", is_flag=True)
@click.option("--log-dir", help="Output build/deployment logs to directory.", default=None)
@click.pass_context
def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, # noqa: C901
request_id, discover, state_file, only_update_state,
dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run,
- include_tags, exclude_tags, log_dir):
+ include_tags, exclude_tags, force_rebuild, log_dir):
if request_id and discover:
print("Cannot specify both --request-id and --discover", file=sys.stderr)
sys.exit(2)
@@ -306,6 +320,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
os.path.abspath(deployment_parent_dir),
kube_config,
image_registry,
+ force_rebuild,
run_log_file
)
status = "DEPLOYED"
diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py
index a4644412..c0a49689 100644
--- a/stack_orchestrator/main.py
+++ b/stack_orchestrator/main.py
@@ -17,7 +17,7 @@ import click
from stack_orchestrator.command_types import CommandOptions
from stack_orchestrator.repos import setup_repositories
-from stack_orchestrator.build import build_containers
+from stack_orchestrator.build import build_containers, fetch_containers
from stack_orchestrator.build import build_npms
from stack_orchestrator.build import build_webapp
from stack_orchestrator.deploy.webapp import (run_webapp,
@@ -52,6 +52,7 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err
cli.add_command(setup_repositories.command, "setup-repositories")
cli.add_command(build_containers.command, "build-containers")
+cli.add_command(fetch_containers.command, "fetch-containers")
cli.add_command(build_npms.command, "build-npms")
cli.add_command(build_webapp.command, "build-webapp")
cli.add_command(run_webapp.command, "run-webapp")