Merge main
This commit is contained in:
commit
c85f31760d
@ -8,6 +8,8 @@ on:
|
|||||||
- '.gitea/workflows/triggers/test-k8s-deploy'
|
- '.gitea/workflows/triggers/test-k8s-deploy'
|
||||||
- '.gitea/workflows/test-k8s-deploy.yml'
|
- '.gitea/workflows/test-k8s-deploy.yml'
|
||||||
- 'tests/k8s-deploy/run-deploy-test.sh'
|
- 'tests/k8s-deploy/run-deploy-test.sh'
|
||||||
|
schedule: # Note: coordinate with other tests to not overload runners at the same time of day
|
||||||
|
- cron: '3 15 * * *'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test:
|
test:
|
||||||
@ -36,6 +38,8 @@ jobs:
|
|||||||
run: ./scripts/create_build_tag_file.sh
|
run: ./scripts/create_build_tag_file.sh
|
||||||
- name: "Build local shiv package"
|
- name: "Build local shiv package"
|
||||||
run: ./scripts/build_shiv_package.sh
|
run: ./scripts/build_shiv_package.sh
|
||||||
|
- name: "Check cgroups version"
|
||||||
|
run: mount | grep cgroup
|
||||||
- name: "Install kind"
|
- name: "Install kind"
|
||||||
run: ./tests/scripts/install-kind.sh
|
run: ./tests/scripts/install-kind.sh
|
||||||
- name: "Install Kubectl"
|
- name: "Install Kubectl"
|
||||||
@ -45,3 +49,4 @@ jobs:
|
|||||||
source /opt/bash-utils/cgroup-helper.sh
|
source /opt/bash-utils/cgroup-helper.sh
|
||||||
join_cgroup
|
join_cgroup
|
||||||
./tests/k8s-deploy/run-deploy-test.sh
|
./tests/k8s-deploy/run-deploy-test.sh
|
||||||
|
|
||||||
|
@ -33,6 +33,7 @@ from stack_orchestrator.base import get_npm_registry_url
|
|||||||
# TODO: find a place for this
|
# TODO: find a place for this
|
||||||
# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
|
# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
|
||||||
|
|
||||||
|
|
||||||
def make_container_build_env(dev_root_path: str,
|
def make_container_build_env(dev_root_path: str,
|
||||||
container_build_dir: str,
|
container_build_dir: str,
|
||||||
debug: bool,
|
debug: bool,
|
||||||
@ -104,6 +105,9 @@ def process_container(stack: str,
|
|||||||
build_command = os.path.join(container_build_dir,
|
build_command = os.path.join(container_build_dir,
|
||||||
"default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
|
"default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
|
||||||
if not dry_run:
|
if not dry_run:
|
||||||
|
# No PATH at all causes failures with podman.
|
||||||
|
if "PATH" not in container_build_env:
|
||||||
|
container_build_env["PATH"] = os.environ["PATH"]
|
||||||
if verbose:
|
if verbose:
|
||||||
print(f"Executing: {build_command} with environment: {container_build_env}")
|
print(f"Executing: {build_command} with environment: {container_build_env}")
|
||||||
build_result = subprocess.run(build_command, shell=True, env=container_build_env)
|
build_result = subprocess.run(build_command, shell=True, env=container_build_env)
|
||||||
@ -119,6 +123,7 @@ def process_container(stack: str,
|
|||||||
else:
|
else:
|
||||||
print("Skipped")
|
print("Skipped")
|
||||||
|
|
||||||
|
|
||||||
@click.command()
|
@click.command()
|
||||||
@click.option('--include', help="only build these containers")
|
@click.option('--include', help="only build these containers")
|
||||||
@click.option('--exclude', help="don\'t build these containers")
|
@click.option('--exclude', help="don\'t build these containers")
|
||||||
|
@ -25,10 +25,11 @@ from decouple import config
|
|||||||
import click
|
import click
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from stack_orchestrator.build import build_containers
|
from stack_orchestrator.build import build_containers
|
||||||
|
from stack_orchestrator.deploy.webapp.util import determine_base_container
|
||||||
|
|
||||||
|
|
||||||
@click.command()
|
@click.command()
|
||||||
@click.option('--base-container', default="cerc/nextjs-base")
|
@click.option('--base-container')
|
||||||
@click.option('--source-repo', help="directory containing the webapp to build", required=True)
|
@click.option('--source-repo', help="directory containing the webapp to build", required=True)
|
||||||
@click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
|
@click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
|
||||||
@click.option("--extra-build-args", help="Supply extra arguments to build")
|
@click.option("--extra-build-args", help="Supply extra arguments to build")
|
||||||
@ -57,6 +58,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
|
|||||||
if not quiet:
|
if not quiet:
|
||||||
print(f'Dev Root is: {dev_root_path}')
|
print(f'Dev Root is: {dev_root_path}')
|
||||||
|
|
||||||
|
if not base_container:
|
||||||
|
base_container = determine_base_container(source_repo)
|
||||||
|
|
||||||
# First build the base container.
|
# First build the base container.
|
||||||
container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
|
container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
|
||||||
force_rebuild, extra_build_args)
|
force_rebuild, extra_build_args)
|
||||||
@ -64,13 +68,12 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
|
|||||||
build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
|
build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
|
||||||
verbose, dry_run, continue_on_error)
|
verbose, dry_run, continue_on_error)
|
||||||
|
|
||||||
|
|
||||||
# Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir.
|
# Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir.
|
||||||
container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
|
container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
|
||||||
container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo)
|
container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo)
|
||||||
container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir,
|
container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir,
|
||||||
base_container.replace("/", "-"),
|
base_container.replace("/", "-"),
|
||||||
"Dockerfile.webapp")
|
"Dockerfile.webapp")
|
||||||
if not tag:
|
if not tag:
|
||||||
webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
|
webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
|
||||||
container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
|
container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
|
||||||
|
@ -30,13 +30,13 @@ RUN \
|
|||||||
|
|
||||||
# [Optional] Uncomment this section to install additional OS packages.
|
# [Optional] Uncomment this section to install additional OS packages.
|
||||||
RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
|
RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
|
||||||
&& apt-get -y install --no-install-recommends jq gettext-base
|
&& apt-get -y install --no-install-recommends jq gettext-base procps
|
||||||
|
|
||||||
# [Optional] Uncomment if you want to install more global node modules
|
# [Optional] Uncomment if you want to install more global node modules
|
||||||
# RUN su node -c "npm install -g <your-package-list-here>"
|
# RUN su node -c "npm install -g <your-package-list-here>"
|
||||||
|
|
||||||
# Expose port for http
|
# Expose port for http
|
||||||
EXPOSE 3000
|
EXPOSE 80
|
||||||
|
|
||||||
COPY /scripts /scripts
|
COPY /scripts /scripts
|
||||||
|
|
||||||
|
@ -58,4 +58,4 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
$CERC_BUILD_TOOL start . -p ${CERC_LISTEN_PORT:-3000}
|
$CERC_BUILD_TOOL start . -- -p ${CERC_LISTEN_PORT:-80}
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
|
# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
|
||||||
# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
|
# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
|
||||||
ARG VARIANT=18-bullseye
|
ARG VARIANT=20-bullseye
|
||||||
FROM node:${VARIANT}
|
FROM node:${VARIANT}
|
||||||
|
|
||||||
ARG USERNAME=node
|
ARG USERNAME=node
|
||||||
@ -28,7 +28,7 @@ RUN \
|
|||||||
|
|
||||||
# [Optional] Uncomment this section to install additional OS packages.
|
# [Optional] Uncomment this section to install additional OS packages.
|
||||||
RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
|
RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
|
||||||
&& apt-get -y install --no-install-recommends jq
|
&& apt-get -y install --no-install-recommends jq gettext-base
|
||||||
|
|
||||||
# [Optional] Uncomment if you want to install an additional version of node using nvm
|
# [Optional] Uncomment if you want to install an additional version of node using nvm
|
||||||
# ARG EXTRA_NODE_VERSION=10
|
# ARG EXTRA_NODE_VERSION=10
|
||||||
@ -37,9 +37,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
|
|||||||
# We do this to get a yq binary from the published container, for the correct architecture we're building here
|
# We do this to get a yq binary from the published container, for the correct architecture we're building here
|
||||||
COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
|
COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
|
||||||
|
|
||||||
RUN mkdir -p /scripts
|
COPY scripts /scripts
|
||||||
COPY ./apply-webapp-config.sh /scripts
|
|
||||||
COPY ./start-serving-app.sh /scripts
|
|
||||||
|
|
||||||
# [Optional] Uncomment if you want to install more global node modules
|
# [Optional] Uncomment if you want to install more global node modules
|
||||||
# RUN su node -c "npm install -g <your-package-list-here>"
|
# RUN su node -c "npm install -g <your-package-list-here>"
|
||||||
|
@ -0,0 +1,11 @@
|
|||||||
|
FROM cerc/webapp-base:local as builder
|
||||||
|
|
||||||
|
ARG CERC_BUILD_TOOL
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
COPY . .
|
||||||
|
RUN rm -rf node_modules build .next*
|
||||||
|
RUN /scripts/build-app.sh /app build /data
|
||||||
|
|
||||||
|
FROM cerc/webapp-base:local
|
||||||
|
COPY --from=builder /data /data
|
@ -1,9 +1,29 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Build cerc/laconic-registry-cli
|
# Build cerc/webapp-base
|
||||||
|
|
||||||
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
|
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
|
||||||
|
|
||||||
# See: https://stackoverflow.com/a/246128/1701505
|
# See: https://stackoverflow.com/a/246128/1701505
|
||||||
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||||
|
|
||||||
docker build -t cerc/webapp-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile ${SCRIPT_DIR}
|
CERC_CONTAINER_BUILD_WORK_DIR=${CERC_CONTAINER_BUILD_WORK_DIR:-$SCRIPT_DIR}
|
||||||
|
CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/Dockerfile}
|
||||||
|
CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/webapp-base:local}
|
||||||
|
|
||||||
|
docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
|
||||||
|
|
||||||
|
if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/webapp-base:local" ]; then
|
||||||
|
cat <<EOF
|
||||||
|
|
||||||
|
#################################################################
|
||||||
|
|
||||||
|
Built host container for $CERC_CONTAINER_BUILD_WORK_DIR with tag:
|
||||||
|
|
||||||
|
$CERC_CONTAINER_BUILD_TAG
|
||||||
|
|
||||||
|
To test locally run:
|
||||||
|
|
||||||
|
laconic-so run-webapp --image $CERC_CONTAINER_BUILD_TAG --env-file /path/to/environment.env
|
||||||
|
|
||||||
|
EOF
|
||||||
|
fi
|
||||||
|
@ -0,0 +1,33 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
|
||||||
|
set -x
|
||||||
|
fi
|
||||||
|
|
||||||
|
WORK_DIR="${1:-./}"
|
||||||
|
|
||||||
|
cd "${WORK_DIR}" || exit 1
|
||||||
|
|
||||||
|
if [ -f ".env" ]; then
|
||||||
|
TMP_ENV=`mktemp`
|
||||||
|
declare -px > $TMP_ENV
|
||||||
|
set -a
|
||||||
|
source .env
|
||||||
|
source $TMP_ENV
|
||||||
|
set +a
|
||||||
|
rm -f $TMP_ENV
|
||||||
|
fi
|
||||||
|
|
||||||
|
for f in $(find . -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do
|
||||||
|
for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},();]' '\n' | egrep -o -e '^"CERC_RUNTIME_ENV_[^\"]+"' -e '^"LACONIC_HOSTED_CONFIG_[^\"]+"'); do
|
||||||
|
orig_name=$(echo -n "${e}" | sed 's/"//g')
|
||||||
|
cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
|
||||||
|
cur_val=$(echo -n "\$${cur_name}" | envsubst)
|
||||||
|
if [ "$CERC_RETAIN_ENV_QUOTES" != "true" ]; then
|
||||||
|
cur_val=$(sed "s/^[\"']//" <<< "$cur_val" | sed "s/[\"']//")
|
||||||
|
fi
|
||||||
|
esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val")
|
||||||
|
echo "$f: $cur_name=$cur_val"
|
||||||
|
sed -i "s/$orig_name/$esc_val/g" $f
|
||||||
|
done
|
||||||
|
done
|
@ -0,0 +1,36 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||||
|
|
||||||
|
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
|
||||||
|
set -x
|
||||||
|
fi
|
||||||
|
|
||||||
|
CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
|
||||||
|
WORK_DIR="${1:-/app}"
|
||||||
|
OUTPUT_DIR="${2:-build}"
|
||||||
|
DEST_DIR="${3:-/data}"
|
||||||
|
|
||||||
|
if [ -f "${WORK_DIR}/package.json" ]; then
|
||||||
|
echo "Building node-based webapp ..."
|
||||||
|
cd "${WORK_DIR}" || exit 1
|
||||||
|
|
||||||
|
if [ -z "$CERC_BUILD_TOOL" ]; then
|
||||||
|
if [ -f "yarn.lock" ]; then
|
||||||
|
CERC_BUILD_TOOL=yarn
|
||||||
|
else
|
||||||
|
CERC_BUILD_TOOL=npm
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
$CERC_BUILD_TOOL install || exit 1
|
||||||
|
$CERC_BUILD_TOOL build || exit 1
|
||||||
|
|
||||||
|
rm -rf "${DEST_DIR}"
|
||||||
|
mv "${WORK_DIR}/${OUTPUT_DIR}" "${DEST_DIR}"
|
||||||
|
else
|
||||||
|
echo "Copying static app ..."
|
||||||
|
mv "${WORK_DIR}" "${DEST_DIR}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
exit 0
|
@ -0,0 +1,15 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
|
||||||
|
set -x
|
||||||
|
fi
|
||||||
|
|
||||||
|
CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}"
|
||||||
|
CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}"
|
||||||
|
|
||||||
|
if [ "true" == "$CERC_ENABLE_CORS" ]; then
|
||||||
|
CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --cors"
|
||||||
|
fi
|
||||||
|
|
||||||
|
/scripts/apply-webapp-config.sh /config/config.yml ${CERC_WEBAPP_FILES_DIR}
|
||||||
|
/scripts/apply-runtime-env.sh ${CERC_WEBAPP_FILES_DIR}
|
||||||
|
http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT:-80} ${CERC_WEBAPP_FILES_DIR}
|
@ -1,9 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
|
|
||||||
set -x
|
|
||||||
fi
|
|
||||||
|
|
||||||
CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}"
|
|
||||||
|
|
||||||
/scripts/apply-webapp-config.sh /config/config.yml ${CERC_WEBAPP_FILES_DIR}
|
|
||||||
http-server -p 80 ${CERC_WEBAPP_FILES_DIR}
|
|
@ -0,0 +1,9 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Build cerc/webapp-deployer-backend
|
||||||
|
|
||||||
|
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
|
||||||
|
|
||||||
|
# See: https://stackoverflow.com/a/246128/1701505
|
||||||
|
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||||
|
|
||||||
|
docker build -t cerc/webapp-deployer-backend:local ${build_command_args} ${CERC_REPO_BASE_DIR}/webapp-deployment-status-api
|
@ -18,7 +18,7 @@ $ laconic-so --stack mainnet-eth build-containers
|
|||||||
|
|
||||||
```
|
```
|
||||||
$ laconic-so --stack mainnet-eth deploy init --map-ports-to-host any-same --output mainnet-eth-spec.yml
|
$ laconic-so --stack mainnet-eth deploy init --map-ports-to-host any-same --output mainnet-eth-spec.yml
|
||||||
$ laconic-so deploy create --spec-file mainnet-eth-spec.yml --deployment-dir mainnet-eth-deployment
|
$ laconic-so deploy --stack mainnet-eth create --spec-file mainnet-eth-spec.yml --deployment-dir mainnet-eth-deployment
|
||||||
```
|
```
|
||||||
## Start the stack
|
## Start the stack
|
||||||
```
|
```
|
||||||
|
@ -0,0 +1,11 @@
|
|||||||
|
version: "1.0"
|
||||||
|
name: webapp-deployer-backend
|
||||||
|
description: "Deployer for webapps"
|
||||||
|
repos:
|
||||||
|
- git.vdb.to:telackey/webapp-deployment-status-api
|
||||||
|
containers:
|
||||||
|
- cerc/webapp-deployer-backend
|
||||||
|
pods:
|
||||||
|
- name: webapp-deployer-backend
|
||||||
|
repository: git.vdb.to:telackey/webapp-deployment-status-api
|
||||||
|
path: ./
|
@ -347,8 +347,8 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
|
|||||||
else:
|
else:
|
||||||
if deployment:
|
if deployment:
|
||||||
compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml")
|
compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml")
|
||||||
pod_pre_start_command = pod["pre_start_command"]
|
pod_pre_start_command = pod.get("pre_start_command")
|
||||||
pod_post_start_command = pod["post_start_command"]
|
pod_post_start_command = pod.get("post_start_command")
|
||||||
script_dir = compose_dir.parent.joinpath("pods", pod_name, "scripts")
|
script_dir = compose_dir.parent.joinpath("pods", pod_name, "scripts")
|
||||||
if pod_pre_start_command is not None:
|
if pod_pre_start_command is not None:
|
||||||
pre_start_commands.append(os.path.join(script_dir, pod_pre_start_command))
|
pre_start_commands.append(os.path.join(script_dir, pod_pre_start_command))
|
||||||
@ -357,8 +357,8 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
|
|||||||
else:
|
else:
|
||||||
pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
|
pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
|
||||||
compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml")
|
compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml")
|
||||||
pod_pre_start_command = pod["pre_start_command"]
|
pod_pre_start_command = pod.get("pre_start_command")
|
||||||
pod_post_start_command = pod["post_start_command"]
|
pod_post_start_command = pod.get("post_start_command")
|
||||||
if pod_pre_start_command is not None:
|
if pod_pre_start_command is not None:
|
||||||
pre_start_commands.append(os.path.join(pod_root_dir, pod_pre_start_command))
|
pre_start_commands.append(os.path.join(pod_root_dir, pod_pre_start_command))
|
||||||
if pod_post_start_command is not None:
|
if pod_post_start_command is not None:
|
||||||
|
@ -54,19 +54,44 @@ def _get_ports(stack):
|
|||||||
|
|
||||||
def _get_named_volumes(stack):
|
def _get_named_volumes(stack):
|
||||||
# Parse the compose files looking for named volumes
|
# Parse the compose files looking for named volumes
|
||||||
named_volumes = []
|
named_volumes = {
|
||||||
|
"rw": [],
|
||||||
|
"ro": []
|
||||||
|
}
|
||||||
parsed_stack = get_parsed_stack_config(stack)
|
parsed_stack = get_parsed_stack_config(stack)
|
||||||
pods = get_pod_list(parsed_stack)
|
pods = get_pod_list(parsed_stack)
|
||||||
yaml = get_yaml()
|
yaml = get_yaml()
|
||||||
|
|
||||||
|
def find_vol_usage(parsed_pod_file, vol):
|
||||||
|
ret = {}
|
||||||
|
if "services" in parsed_pod_file:
|
||||||
|
for svc_name, svc in parsed_pod_file["services"].items():
|
||||||
|
if "volumes" in svc:
|
||||||
|
for svc_volume in svc["volumes"]:
|
||||||
|
parts = svc_volume.split(":")
|
||||||
|
if parts[0] == vol:
|
||||||
|
ret[svc_name] = {
|
||||||
|
"volume": parts[0],
|
||||||
|
"mount": parts[1],
|
||||||
|
"options": parts[2] if len(parts) == 3 else None
|
||||||
|
}
|
||||||
|
return ret
|
||||||
|
|
||||||
for pod in pods:
|
for pod in pods:
|
||||||
pod_file_path = get_pod_file_path(parsed_stack, pod)
|
pod_file_path = get_pod_file_path(parsed_stack, pod)
|
||||||
parsed_pod_file = yaml.load(open(pod_file_path, "r"))
|
parsed_pod_file = yaml.load(open(pod_file_path, "r"))
|
||||||
if "volumes" in parsed_pod_file:
|
if "volumes" in parsed_pod_file:
|
||||||
volumes = parsed_pod_file["volumes"]
|
volumes = parsed_pod_file["volumes"]
|
||||||
for volume in volumes.keys():
|
for volume in volumes.keys():
|
||||||
# Volume definition looks like:
|
for vu in find_vol_usage(parsed_pod_file, volume).values():
|
||||||
# 'laconicd-data': None
|
read_only = vu["options"] == "ro"
|
||||||
named_volumes.append(volume)
|
if read_only:
|
||||||
|
if vu["volume"] not in named_volumes["rw"] and vu["volume"] not in named_volumes["ro"]:
|
||||||
|
named_volumes["ro"].append(vu["volume"])
|
||||||
|
else:
|
||||||
|
if vu["volume"] not in named_volumes["rw"]:
|
||||||
|
named_volumes["rw"].append(vu["volume"])
|
||||||
|
|
||||||
return named_volumes
|
return named_volumes
|
||||||
|
|
||||||
|
|
||||||
@ -98,12 +123,24 @@ def _fixup_pod_file(pod, spec, compose_dir):
|
|||||||
_create_bind_dir_if_relative(volume, volume_spec, compose_dir)
|
_create_bind_dir_if_relative(volume, volume_spec, compose_dir)
|
||||||
new_volume_spec = {"driver": "local",
|
new_volume_spec = {"driver": "local",
|
||||||
"driver_opts": {
|
"driver_opts": {
|
||||||
"type": "none",
|
"type": "none",
|
||||||
"device": volume_spec_fixedup,
|
"device": volume_spec_fixedup,
|
||||||
"o": "bind"
|
"o": "bind"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pod["volumes"][volume] = new_volume_spec
|
pod["volumes"][volume] = new_volume_spec
|
||||||
|
|
||||||
|
# Fix up configmaps
|
||||||
|
if "configmaps" in spec:
|
||||||
|
spec_cfgmaps = spec["configmaps"]
|
||||||
|
if "volumes" in pod:
|
||||||
|
pod_volumes = pod["volumes"]
|
||||||
|
for volume in pod_volumes.keys():
|
||||||
|
if volume in spec_cfgmaps:
|
||||||
|
volume_cfg = spec_cfgmaps[volume]
|
||||||
|
# Just make the dir (if necessary)
|
||||||
|
_create_bind_dir_if_relative(volume, volume_cfg, compose_dir)
|
||||||
|
|
||||||
# Fix up ports
|
# Fix up ports
|
||||||
if "network" in spec and "ports" in spec["network"]:
|
if "network" in spec and "ports" in spec["network"]:
|
||||||
spec_ports = spec["network"]["ports"]
|
spec_ports = spec["network"]["ports"]
|
||||||
@ -319,9 +356,18 @@ def init_operation(deploy_command_context, stack, deployer_type, config,
|
|||||||
named_volumes = _get_named_volumes(stack)
|
named_volumes = _get_named_volumes(stack)
|
||||||
if named_volumes:
|
if named_volumes:
|
||||||
volume_descriptors = {}
|
volume_descriptors = {}
|
||||||
for named_volume in named_volumes:
|
configmap_descriptors = {}
|
||||||
|
for named_volume in named_volumes["rw"]:
|
||||||
volume_descriptors[named_volume] = f"./data/{named_volume}"
|
volume_descriptors[named_volume] = f"./data/{named_volume}"
|
||||||
spec_file_content["volumes"] = volume_descriptors
|
for named_volume in named_volumes["ro"]:
|
||||||
|
if "k8s" in deployer_type and "config" in named_volume:
|
||||||
|
configmap_descriptors[named_volume] = f"./data/{named_volume}"
|
||||||
|
else:
|
||||||
|
volume_descriptors[named_volume] = f"./data/{named_volume}"
|
||||||
|
if volume_descriptors:
|
||||||
|
spec_file_content["volumes"] = volume_descriptors
|
||||||
|
if configmap_descriptors:
|
||||||
|
spec_file_content["configmaps"] = configmap_descriptors
|
||||||
|
|
||||||
if opts.o.debug:
|
if opts.o.debug:
|
||||||
print(f"Creating spec file for stack: {stack} with content: {spec_file_content}")
|
print(f"Creating spec file for stack: {stack} with content: {spec_file_content}")
|
||||||
|
@ -31,7 +31,8 @@ def _image_needs_pushed(image: str):
|
|||||||
|
|
||||||
def remote_tag_for_image(image: str, remote_repo_url: str):
|
def remote_tag_for_image(image: str, remote_repo_url: str):
|
||||||
# Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
|
# Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
|
||||||
(org, image_name_with_version) = image.split("/")
|
major_parts = image.split("/", 2)
|
||||||
|
image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
|
||||||
(image_name, image_version) = image_name_with_version.split(":")
|
(image_name, image_version) = image_name_with_version.split(":")
|
||||||
if image_version == "local":
|
if image_version == "local":
|
||||||
return f"{remote_repo_url}/{image_name}:deploy"
|
return f"{remote_repo_url}/{image_name}:deploy"
|
||||||
|
@ -13,6 +13,8 @@
|
|||||||
# You should have received a copy of the GNU Affero General Public License
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
from kubernetes import client
|
from kubernetes import client
|
||||||
from typing import Any, List, Set
|
from typing import Any, List, Set
|
||||||
|
|
||||||
@ -112,9 +114,10 @@ class ClusterInfo:
|
|||||||
services = pod["services"]
|
services = pod["services"]
|
||||||
for service_name in services:
|
for service_name in services:
|
||||||
service_info = services[service_name]
|
service_info = services[service_name]
|
||||||
port = int(service_info["ports"][0])
|
if "ports" in service_info:
|
||||||
if opts.o.debug:
|
port = int(service_info["ports"][0])
|
||||||
print(f"service port: {port}")
|
if opts.o.debug:
|
||||||
|
print(f"service port: {port}")
|
||||||
service = client.V1Service(
|
service = client.V1Service(
|
||||||
metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"),
|
metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"),
|
||||||
spec=client.V1ServiceSpec(
|
spec=client.V1ServiceSpec(
|
||||||
@ -130,30 +133,70 @@ class ClusterInfo:
|
|||||||
|
|
||||||
def get_pvcs(self):
|
def get_pvcs(self):
|
||||||
result = []
|
result = []
|
||||||
volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
|
spec_volumes = self.spec.get_volumes()
|
||||||
|
named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
|
||||||
if opts.o.debug:
|
if opts.o.debug:
|
||||||
print(f"Volumes: {volumes}")
|
print(f"Spec Volumes: {spec_volumes}")
|
||||||
for volume_name in volumes:
|
print(f"Named Volumes: {named_volumes}")
|
||||||
|
for volume_name in spec_volumes:
|
||||||
|
if volume_name not in named_volumes:
|
||||||
|
if opts.o.debug:
|
||||||
|
print(f"{volume_name} not in pod files")
|
||||||
|
continue
|
||||||
spec = client.V1PersistentVolumeClaimSpec(
|
spec = client.V1PersistentVolumeClaimSpec(
|
||||||
access_modes=["ReadWriteOnce"],
|
access_modes=["ReadWriteOnce"],
|
||||||
storage_class_name="manual",
|
storage_class_name="manual",
|
||||||
resources=client.V1ResourceRequirements(
|
resources=client.V1ResourceRequirements(
|
||||||
requests={"storage": "2Gi"}
|
requests={"storage": "2Gi"}
|
||||||
),
|
),
|
||||||
volume_name=volume_name
|
volume_name=f"{self.app_name}-{volume_name}"
|
||||||
)
|
)
|
||||||
pvc = client.V1PersistentVolumeClaim(
|
pvc = client.V1PersistentVolumeClaim(
|
||||||
metadata=client.V1ObjectMeta(name=volume_name,
|
metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}",
|
||||||
labels={"volume-label": volume_name}),
|
labels={"volume-label": f"{self.app_name}-{volume_name}"}),
|
||||||
spec=spec,
|
spec=spec,
|
||||||
)
|
)
|
||||||
result.append(pvc)
|
result.append(pvc)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
def get_configmaps(self):
|
||||||
|
result = []
|
||||||
|
spec_configmaps = self.spec.get_configmaps()
|
||||||
|
named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
|
||||||
|
for cfg_map_name, cfg_map_path in spec_configmaps.items():
|
||||||
|
if cfg_map_name not in named_volumes:
|
||||||
|
if opts.o.debug:
|
||||||
|
print(f"{cfg_map_name} not in pod files")
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not cfg_map_path.startswith("/"):
|
||||||
|
cfg_map_path = os.path.join(os.path.dirname(self.spec.file_path), cfg_map_path)
|
||||||
|
|
||||||
|
# Read in all the files at a single-level of the directory. This mimics the behavior
|
||||||
|
# of `kubectl create configmap foo --from-file=/path/to/dir`
|
||||||
|
data = {}
|
||||||
|
for f in os.listdir(cfg_map_path):
|
||||||
|
full_path = os.path.join(cfg_map_path, f)
|
||||||
|
if os.path.isfile(full_path):
|
||||||
|
data[f] = open(full_path, 'rt').read()
|
||||||
|
|
||||||
|
spec = client.V1ConfigMap(
|
||||||
|
metadata=client.V1ObjectMeta(name=f"{self.app_name}-{cfg_map_name}",
|
||||||
|
labels={"configmap-label": cfg_map_name}),
|
||||||
|
data=data
|
||||||
|
)
|
||||||
|
result.append(spec)
|
||||||
|
return result
|
||||||
|
|
||||||
def get_pvs(self):
|
def get_pvs(self):
|
||||||
result = []
|
result = []
|
||||||
volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
|
spec_volumes = self.spec.get_volumes()
|
||||||
for volume_name in volumes:
|
named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
|
||||||
|
for volume_name in spec_volumes:
|
||||||
|
if volume_name not in named_volumes:
|
||||||
|
if opts.o.debug:
|
||||||
|
print(f"{volume_name} not in pod files")
|
||||||
|
continue
|
||||||
spec = client.V1PersistentVolumeSpec(
|
spec = client.V1PersistentVolumeSpec(
|
||||||
storage_class_name="manual",
|
storage_class_name="manual",
|
||||||
access_modes=["ReadWriteOnce"],
|
access_modes=["ReadWriteOnce"],
|
||||||
@ -161,8 +204,8 @@ class ClusterInfo:
|
|||||||
host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name))
|
host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name))
|
||||||
)
|
)
|
||||||
pv = client.V1PersistentVolume(
|
pv = client.V1PersistentVolume(
|
||||||
metadata=client.V1ObjectMeta(name=volume_name,
|
metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}",
|
||||||
labels={"volume-label": volume_name}),
|
labels={"volume-label": f"{self.app_name}-{volume_name}"}),
|
||||||
spec=spec,
|
spec=spec,
|
||||||
)
|
)
|
||||||
result.append(pv)
|
result.append(pv)
|
||||||
@ -178,10 +221,11 @@ class ClusterInfo:
|
|||||||
container_name = service_name
|
container_name = service_name
|
||||||
service_info = services[service_name]
|
service_info = services[service_name]
|
||||||
image = service_info["image"]
|
image = service_info["image"]
|
||||||
port = int(service_info["ports"][0])
|
if "ports" in service_info:
|
||||||
if opts.o.debug:
|
port = int(service_info["ports"][0])
|
||||||
print(f"image: {image}")
|
if opts.o.debug:
|
||||||
print(f"service port: {port}")
|
print(f"image: {image}")
|
||||||
|
print(f"service port: {port}")
|
||||||
# Re-write the image tag for remote deployment
|
# Re-write the image tag for remote deployment
|
||||||
image_to_use = remote_tag_for_image(
|
image_to_use = remote_tag_for_image(
|
||||||
image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
|
image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
|
||||||
@ -195,11 +239,11 @@ class ClusterInfo:
|
|||||||
volume_mounts=volume_mounts,
|
volume_mounts=volume_mounts,
|
||||||
resources=client.V1ResourceRequirements(
|
resources=client.V1ResourceRequirements(
|
||||||
requests={"cpu": "100m", "memory": "200Mi"},
|
requests={"cpu": "100m", "memory": "200Mi"},
|
||||||
limits={"cpu": "500m", "memory": "500Mi"},
|
limits={"cpu": "1000m", "memory": "2000Mi"},
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
containers.append(container)
|
containers.append(container)
|
||||||
volumes = volumes_for_pod_files(self.parsed_pod_yaml_map)
|
volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec, self.app_name)
|
||||||
image_pull_secrets = [client.V1LocalObjectReference(name="laconic-registry")]
|
image_pull_secrets = [client.V1LocalObjectReference(name="laconic-registry")]
|
||||||
template = client.V1PodTemplateSpec(
|
template = client.V1PodTemplateSpec(
|
||||||
metadata=client.V1ObjectMeta(labels={"app": self.app_name}),
|
metadata=client.V1ObjectMeta(labels={"app": self.app_name}),
|
||||||
|
@ -1,5 +1,4 @@
|
|||||||
# Copyright © 2023 Vulcanize
|
# Copyright © 2023 Vulcanize
|
||||||
|
|
||||||
# This program is free software: you can redistribute it and/or modify
|
# This program is free software: you can redistribute it and/or modify
|
||||||
# it under the terms of the GNU Affero General Public License as published by
|
# it under the terms of the GNU Affero General Public License as published by
|
||||||
# the Free Software Foundation, either version 3 of the License, or
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
@ -110,6 +109,20 @@ class K8sDeployer(Deployer):
|
|||||||
if opts.o.debug:
|
if opts.o.debug:
|
||||||
print("PVCs created:")
|
print("PVCs created:")
|
||||||
print(f"{pvc_resp}")
|
print(f"{pvc_resp}")
|
||||||
|
|
||||||
|
# Figure out the ConfigMaps for this deployment
|
||||||
|
config_maps = self.cluster_info.get_configmaps()
|
||||||
|
for cfg_map in config_maps:
|
||||||
|
if opts.o.debug:
|
||||||
|
print(f"Sending this ConfigMap: {cfg_map}")
|
||||||
|
cfg_rsp = self.core_api.create_namespaced_config_map(
|
||||||
|
body=cfg_map,
|
||||||
|
namespace=self.k8s_namespace
|
||||||
|
)
|
||||||
|
if opts.o.debug:
|
||||||
|
print("ConfigMap created:")
|
||||||
|
print(f"{cfg_rsp}")
|
||||||
|
|
||||||
# Process compose files into a Deployment
|
# Process compose files into a Deployment
|
||||||
deployment = self.cluster_info.get_deployment(image_pull_policy=None if self.is_kind() else "Always")
|
deployment = self.cluster_info.get_deployment(image_pull_policy=None if self.is_kind() else "Always")
|
||||||
# Create the k8s objects
|
# Create the k8s objects
|
||||||
@ -135,17 +148,21 @@ class K8sDeployer(Deployer):
|
|||||||
if not self.is_kind():
|
if not self.is_kind():
|
||||||
ingress: client.V1Ingress = self.cluster_info.get_ingress()
|
ingress: client.V1Ingress = self.cluster_info.get_ingress()
|
||||||
|
|
||||||
if opts.o.debug:
|
if ingress:
|
||||||
print(f"Sending this ingress: {ingress}")
|
if opts.o.debug:
|
||||||
ingress_resp = self.networking_api.create_namespaced_ingress(
|
print(f"Sending this ingress: {ingress}")
|
||||||
namespace=self.k8s_namespace,
|
ingress_resp = self.networking_api.create_namespaced_ingress(
|
||||||
body=ingress
|
namespace=self.k8s_namespace,
|
||||||
)
|
body=ingress
|
||||||
if opts.o.debug:
|
)
|
||||||
print("Ingress created:")
|
if opts.o.debug:
|
||||||
print(f"{ingress_resp}")
|
print("Ingress created:")
|
||||||
|
print(f"{ingress_resp}")
|
||||||
|
else:
|
||||||
|
if opts.o.debug:
|
||||||
|
print("No ingress configured")
|
||||||
|
|
||||||
def down(self, timeout, volumes):
|
def down(self, timeout, volumes): # noqa: C901
|
||||||
self.connect_api()
|
self.connect_api()
|
||||||
# Delete the k8s objects
|
# Delete the k8s objects
|
||||||
# Create the host-path-mounted PVs for this deployment
|
# Create the host-path-mounted PVs for this deployment
|
||||||
@ -175,6 +192,22 @@ class K8sDeployer(Deployer):
|
|||||||
print(f"{pvc_resp}")
|
print(f"{pvc_resp}")
|
||||||
except client.exceptions.ApiException as e:
|
except client.exceptions.ApiException as e:
|
||||||
_check_delete_exception(e)
|
_check_delete_exception(e)
|
||||||
|
|
||||||
|
# Figure out the ConfigMaps for this deployment
|
||||||
|
cfg_maps = self.cluster_info.get_configmaps()
|
||||||
|
for cfg_map in cfg_maps:
|
||||||
|
if opts.o.debug:
|
||||||
|
print(f"Deleting this ConfigMap: {cfg_map}")
|
||||||
|
try:
|
||||||
|
cfg_map_resp = self.core_api.delete_namespaced_config_map(
|
||||||
|
name=cfg_map.metadata.name, namespace=self.k8s_namespace
|
||||||
|
)
|
||||||
|
if opts.o.debug:
|
||||||
|
print("ConfigMap deleted:")
|
||||||
|
print(f"{cfg_map_resp}")
|
||||||
|
except client.exceptions.ApiException as e:
|
||||||
|
_check_delete_exception(e)
|
||||||
|
|
||||||
deployment = self.cluster_info.get_deployment()
|
deployment = self.cluster_info.get_deployment()
|
||||||
if opts.o.debug:
|
if opts.o.debug:
|
||||||
print(f"Deleting this deployment: {deployment}")
|
print(f"Deleting this deployment: {deployment}")
|
||||||
@ -198,14 +231,18 @@ class K8sDeployer(Deployer):
|
|||||||
|
|
||||||
if not self.is_kind():
|
if not self.is_kind():
|
||||||
ingress: client.V1Ingress = self.cluster_info.get_ingress()
|
ingress: client.V1Ingress = self.cluster_info.get_ingress()
|
||||||
if opts.o.debug:
|
if ingress:
|
||||||
print(f"Deleting this ingress: {ingress}")
|
if opts.o.debug:
|
||||||
try:
|
print(f"Deleting this ingress: {ingress}")
|
||||||
self.networking_api.delete_namespaced_ingress(
|
try:
|
||||||
name=ingress.metadata.name, namespace=self.k8s_namespace
|
self.networking_api.delete_namespaced_ingress(
|
||||||
)
|
name=ingress.metadata.name, namespace=self.k8s_namespace
|
||||||
except client.exceptions.ApiException as e:
|
)
|
||||||
_check_delete_exception(e)
|
except client.exceptions.ApiException as e:
|
||||||
|
_check_delete_exception(e)
|
||||||
|
else:
|
||||||
|
if opts.o.debug:
|
||||||
|
print("No ingress to delete")
|
||||||
|
|
||||||
if self.is_kind():
|
if self.is_kind():
|
||||||
# Destroy the kind cluster
|
# Destroy the kind cluster
|
||||||
|
@ -73,7 +73,7 @@ def named_volumes_from_pod_files(parsed_pod_files):
|
|||||||
parsed_pod_file = parsed_pod_files[pod]
|
parsed_pod_file = parsed_pod_files[pod]
|
||||||
if "volumes" in parsed_pod_file:
|
if "volumes" in parsed_pod_file:
|
||||||
volumes = parsed_pod_file["volumes"]
|
volumes = parsed_pod_file["volumes"]
|
||||||
for volume in volumes.keys():
|
for volume, value in volumes.items():
|
||||||
# Volume definition looks like:
|
# Volume definition looks like:
|
||||||
# 'laconicd-data': None
|
# 'laconicd-data': None
|
||||||
named_volumes.append(volume)
|
named_volumes.append(volume)
|
||||||
@ -103,24 +103,32 @@ def volume_mounts_for_service(parsed_pod_files, service):
|
|||||||
mount_split = mount_string.split(":")
|
mount_split = mount_string.split(":")
|
||||||
volume_name = mount_split[0]
|
volume_name = mount_split[0]
|
||||||
mount_path = mount_split[1]
|
mount_path = mount_split[1]
|
||||||
|
mount_options = mount_split[2] if len(mount_split) == 3 else None
|
||||||
if opts.o.debug:
|
if opts.o.debug:
|
||||||
print(f"volumne_name: {volume_name}")
|
print(f"volumne_name: {volume_name}")
|
||||||
print(f"mount path: {mount_path}")
|
print(f"mount path: {mount_path}")
|
||||||
volume_device = client.V1VolumeMount(mount_path=mount_path, name=volume_name)
|
print(f"mount options: {mount_options}")
|
||||||
|
volume_device = client.V1VolumeMount(
|
||||||
|
mount_path=mount_path, name=volume_name, read_only="ro" == mount_options)
|
||||||
result.append(volume_device)
|
result.append(volume_device)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def volumes_for_pod_files(parsed_pod_files):
|
def volumes_for_pod_files(parsed_pod_files, spec, app_name):
|
||||||
result = []
|
result = []
|
||||||
for pod in parsed_pod_files:
|
for pod in parsed_pod_files:
|
||||||
parsed_pod_file = parsed_pod_files[pod]
|
parsed_pod_file = parsed_pod_files[pod]
|
||||||
if "volumes" in parsed_pod_file:
|
if "volumes" in parsed_pod_file:
|
||||||
volumes = parsed_pod_file["volumes"]
|
volumes = parsed_pod_file["volumes"]
|
||||||
for volume_name in volumes.keys():
|
for volume_name in volumes.keys():
|
||||||
claim = client.V1PersistentVolumeClaimVolumeSource(claim_name=volume_name)
|
if volume_name in spec.get_configmaps():
|
||||||
volume = client.V1Volume(name=volume_name, persistent_volume_claim=claim)
|
config_map = client.V1ConfigMapVolumeSource(name=f"{app_name}-{volume_name}")
|
||||||
result.append(volume)
|
volume = client.V1Volume(name=volume_name, config_map=config_map)
|
||||||
|
result.append(volume)
|
||||||
|
else:
|
||||||
|
claim = client.V1PersistentVolumeClaimVolumeSource(claim_name=f"{app_name}-{volume_name}")
|
||||||
|
volume = client.V1Volume(name=volume_name, persistent_volume_claim=claim)
|
||||||
|
result.append(volume)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
@ -173,7 +181,7 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir):
|
|||||||
volume_definitions.append(
|
volume_definitions.append(
|
||||||
f" - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n"
|
f" - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n"
|
||||||
f" containerPath: {get_node_pv_mount_path(volume_name)}\n"
|
f" containerPath: {get_node_pv_mount_path(volume_name)}\n"
|
||||||
)
|
)
|
||||||
return (
|
return (
|
||||||
"" if len(volume_definitions) == 0 else (
|
"" if len(volume_definitions) == 0 else (
|
||||||
" extraMounts:\n"
|
" extraMounts:\n"
|
||||||
|
@ -22,6 +22,7 @@ from stack_orchestrator import constants
|
|||||||
class Spec:
|
class Spec:
|
||||||
|
|
||||||
obj: typing.Any
|
obj: typing.Any
|
||||||
|
file_path: Path
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
pass
|
pass
|
||||||
@ -29,12 +30,23 @@ class Spec:
|
|||||||
def init_from_file(self, file_path: Path):
|
def init_from_file(self, file_path: Path):
|
||||||
with file_path:
|
with file_path:
|
||||||
self.obj = get_yaml().load(open(file_path, "r"))
|
self.obj = get_yaml().load(open(file_path, "r"))
|
||||||
|
self.file_path = file_path
|
||||||
|
|
||||||
def get_image_registry(self):
|
def get_image_registry(self):
|
||||||
return (self.obj[constants.image_resigtry_key]
|
return (self.obj[constants.image_resigtry_key]
|
||||||
if self.obj and constants.image_resigtry_key in self.obj
|
if self.obj and constants.image_resigtry_key in self.obj
|
||||||
else None)
|
else None)
|
||||||
|
|
||||||
|
def get_volumes(self):
|
||||||
|
return (self.obj["volumes"]
|
||||||
|
if self.obj and "volumes" in self.obj
|
||||||
|
else {})
|
||||||
|
|
||||||
|
def get_configmaps(self):
|
||||||
|
return (self.obj["configmaps"]
|
||||||
|
if self.obj and "configmaps" in self.obj
|
||||||
|
else {})
|
||||||
|
|
||||||
def get_http_proxy(self):
|
def get_http_proxy(self):
|
||||||
return (self.obj[constants.network_key][constants.http_proxy_key]
|
return (self.obj[constants.network_key][constants.http_proxy_key]
|
||||||
if self.obj and constants.network_key in self.obj
|
if self.obj and constants.network_key in self.obj
|
||||||
|
@ -59,8 +59,8 @@ def process_app_deployment_request(
|
|||||||
dns_record = laconic.get_record(dns_crn)
|
dns_record = laconic.get_record(dns_crn)
|
||||||
if dns_record:
|
if dns_record:
|
||||||
matched_owner = match_owner(app_deployment_request, dns_record)
|
matched_owner = match_owner(app_deployment_request, dns_record)
|
||||||
if not matched_owner and dns_record.request:
|
if not matched_owner and dns_record.attributes.request:
|
||||||
matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.request, require=True))
|
matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.attributes.request, require=True))
|
||||||
|
|
||||||
if matched_owner:
|
if matched_owner:
|
||||||
print("Matched DnsRecord ownership:", matched_owner)
|
print("Matched DnsRecord ownership:", matched_owner)
|
||||||
@ -136,13 +136,17 @@ def load_known_requests(filename):
|
|||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
|
||||||
def dump_known_requests(filename, requests):
|
def dump_known_requests(filename, requests, status="SEEN"):
|
||||||
if not filename:
|
if not filename:
|
||||||
return
|
return
|
||||||
known_requests = load_known_requests(filename)
|
known_requests = load_known_requests(filename)
|
||||||
for r in requests:
|
for r in requests:
|
||||||
known_requests[r.id] = r.createTime
|
known_requests[r.id] = {
|
||||||
json.dump(known_requests, open(filename, "w"))
|
"createTime": r.createTime,
|
||||||
|
"status": status
|
||||||
|
}
|
||||||
|
with open(filename, "w") as f:
|
||||||
|
json.dump(known_requests, f)
|
||||||
|
|
||||||
|
|
||||||
@click.command()
|
@click.command()
|
||||||
@ -201,6 +205,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
|
|||||||
requests.reverse()
|
requests.reverse()
|
||||||
requests_by_name = {}
|
requests_by_name = {}
|
||||||
for r in requests:
|
for r in requests:
|
||||||
|
# TODO: Do this _after_ filtering deployments and cancellations to minimize round trips.
|
||||||
app = laconic.get_record(r.attributes.application)
|
app = laconic.get_record(r.attributes.application)
|
||||||
if not app:
|
if not app:
|
||||||
print("Skipping request %s, cannot locate app." % r.id)
|
print("Skipping request %s, cannot locate app." % r.id)
|
||||||
@ -256,6 +261,8 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
|
|||||||
|
|
||||||
if not dry_run:
|
if not dry_run:
|
||||||
for r in requests_to_execute:
|
for r in requests_to_execute:
|
||||||
|
dump_known_requests(state_file, [r], "DEPLOYING")
|
||||||
|
status = "ERROR"
|
||||||
try:
|
try:
|
||||||
process_app_deployment_request(
|
process_app_deployment_request(
|
||||||
ctx,
|
ctx,
|
||||||
@ -268,5 +275,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
|
|||||||
kube_config,
|
kube_config,
|
||||||
image_registry
|
image_registry
|
||||||
)
|
)
|
||||||
|
status = "DEPLOYED"
|
||||||
finally:
|
finally:
|
||||||
dump_known_requests(state_file, [r])
|
dump_known_requests(state_file, [r], status)
|
||||||
|
@ -27,7 +27,7 @@ from dotenv import dotenv_values
|
|||||||
from stack_orchestrator import constants
|
from stack_orchestrator import constants
|
||||||
from stack_orchestrator.deploy.deployer_factory import getDeployer
|
from stack_orchestrator.deploy.deployer_factory import getDeployer
|
||||||
|
|
||||||
WEBAPP_PORT = 3000
|
WEBAPP_PORT = 80
|
||||||
|
|
||||||
|
|
||||||
@click.command()
|
@click.command()
|
||||||
|
@ -40,8 +40,8 @@ def process_app_removal_request(ctx,
|
|||||||
matched_owner = match_owner(app_removal_request, deployment_record, dns_record)
|
matched_owner = match_owner(app_removal_request, deployment_record, dns_record)
|
||||||
|
|
||||||
# Or of the original deployment request.
|
# Or of the original deployment request.
|
||||||
if not matched_owner and deployment_record.request:
|
if not matched_owner and deployment_record.attributes.request:
|
||||||
matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.request, require=True))
|
matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.attributes.request, require=True))
|
||||||
|
|
||||||
if matched_owner:
|
if matched_owner:
|
||||||
print("Matched deployment ownership:", matched_owner)
|
print("Matched deployment ownership:", matched_owner)
|
||||||
|
@ -195,6 +195,23 @@ def file_hash(filename):
|
|||||||
return hashlib.sha1(open(filename).read().encode()).hexdigest()
|
return hashlib.sha1(open(filename).read().encode()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def determine_base_container(clone_dir, app_type="webapp"):
|
||||||
|
if not app_type or not app_type.startswith("webapp"):
|
||||||
|
raise Exception(f"Unsupported app_type {app_type}")
|
||||||
|
|
||||||
|
base_container = "cerc/webapp-base"
|
||||||
|
if app_type == "webapp/next":
|
||||||
|
base_container = "cerc/nextjs-base"
|
||||||
|
elif app_type == "webapp":
|
||||||
|
pkg_json_path = os.path.join(clone_dir, "package.json")
|
||||||
|
if os.path.exists(pkg_json_path):
|
||||||
|
pkg_json = json.load(open(pkg_json_path))
|
||||||
|
if "next" in pkg_json.get("dependencies", {}):
|
||||||
|
base_container = "cerc/nextjs-base"
|
||||||
|
|
||||||
|
return base_container
|
||||||
|
|
||||||
|
|
||||||
def build_container_image(app_record, tag, extra_build_args=[]):
|
def build_container_image(app_record, tag, extra_build_args=[]):
|
||||||
tmpdir = tempfile.mkdtemp()
|
tmpdir = tempfile.mkdtemp()
|
||||||
|
|
||||||
@ -216,8 +233,15 @@ def build_container_image(app_record, tag, extra_build_args=[]):
|
|||||||
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir])
|
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir])
|
||||||
result.check_returncode()
|
result.check_returncode()
|
||||||
|
|
||||||
|
base_container = determine_base_container(clone_dir, app_record.attributes.app_type)
|
||||||
|
|
||||||
print("Building webapp ...")
|
print("Building webapp ...")
|
||||||
build_command = [sys.argv[0], "build-webapp", "--source-repo", clone_dir, "--tag", tag]
|
build_command = [
|
||||||
|
sys.argv[0], "build-webapp",
|
||||||
|
"--source-repo", clone_dir,
|
||||||
|
"--tag", tag,
|
||||||
|
"--base-container", base_container
|
||||||
|
]
|
||||||
if extra_build_args:
|
if extra_build_args:
|
||||||
build_command.append("--extra-build-args")
|
build_command.append("--extra-build-args")
|
||||||
build_command.append(" ".join(extra_build_args))
|
build_command.append(" ".join(extra_build_args))
|
||||||
|
@ -120,7 +120,8 @@ fi
|
|||||||
# Stop then start again and check the volume was preserved
|
# Stop then start again and check the volume was preserved
|
||||||
$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
|
$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
|
||||||
# Sleep a bit just in case
|
# Sleep a bit just in case
|
||||||
sleep 2
|
# sleep for longer to check if that's why the subsequent create cluster fails
|
||||||
|
sleep 20
|
||||||
$TEST_TARGET_SO deployment --dir $test_deployment_dir start
|
$TEST_TARGET_SO deployment --dir $test_deployment_dir start
|
||||||
wait_for_pods_started
|
wait_for_pods_started
|
||||||
wait_for_log_output
|
wait_for_log_output
|
||||||
|
@ -30,14 +30,14 @@ CHECK="SPECIAL_01234567890_TEST_STRING"
|
|||||||
|
|
||||||
set +e
|
set +e
|
||||||
|
|
||||||
CONTAINER_ID=$(docker run -p 3000:3000 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local)
|
CONTAINER_ID=$(docker run -p 3000:80 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local)
|
||||||
sleep 3
|
sleep 3
|
||||||
wget -t 7 -O test.before -m http://localhost:3000
|
wget -t 7 -O test.before -m http://localhost:3000
|
||||||
|
|
||||||
docker logs $CONTAINER_ID
|
docker logs $CONTAINER_ID
|
||||||
docker remove -f $CONTAINER_ID
|
docker remove -f $CONTAINER_ID
|
||||||
|
|
||||||
CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local)
|
CONTAINER_ID=$(docker run -p 3000:80 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local)
|
||||||
sleep 3
|
sleep 3
|
||||||
wget -t 7 -O test.after -m http://localhost:3000
|
wget -t 7 -O test.after -m http://localhost:3000
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user