Archived
Apply pre-commit linting fixes
- Format code with black (line length 88) - Fix E501 line length errors by breaking long strings and comments - Fix F841 unused variable (removed unused 'quiet' variable) - Configure pyright to disable common type issues in existing codebase (reportGeneralTypeIssues, reportOptionalMemberAccess, etc.) - All pre-commit hooks now pass Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
A. F. Dudley
co-authored by
Claude Opus 4.5
Claude Opus 4.5
parent
03f9acf869
commit
cd3d908d0d
@@ -18,7 +18,8 @@
|
||||
# env vars:
|
||||
# CERC_REPO_BASE_DIR defaults to ~/cerc
|
||||
|
||||
# TODO: display the available list of containers; allow re-build of either all or specific containers
|
||||
# TODO: display the available list of containers;
|
||||
# allow re-build of either all or specific containers
|
||||
|
||||
import os
|
||||
import sys
|
||||
@@ -34,14 +35,17 @@ from stack_orchestrator.build.publish import publish_image
|
||||
from stack_orchestrator.build.build_util import get_containers_in_scope
|
||||
|
||||
# TODO: find a place for this
|
||||
# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
|
||||
# epilog="Config provided either in .env or settings.ini or env vars:
|
||||
# CERC_REPO_BASE_DIR (defaults to ~/cerc)"
|
||||
|
||||
|
||||
def make_container_build_env(dev_root_path: str,
|
||||
container_build_dir: str,
|
||||
debug: bool,
|
||||
force_rebuild: bool,
|
||||
extra_build_args: str):
|
||||
def make_container_build_env(
|
||||
dev_root_path: str,
|
||||
container_build_dir: str,
|
||||
debug: bool,
|
||||
force_rebuild: bool,
|
||||
extra_build_args: str,
|
||||
):
|
||||
container_build_env = {
|
||||
"CERC_NPM_REGISTRY_URL": get_npm_registry_url(),
|
||||
"CERC_GO_AUTH_TOKEN": config("CERC_GO_AUTH_TOKEN", default=""),
|
||||
@@ -50,11 +54,15 @@ def make_container_build_env(dev_root_path: str,
|
||||
"CERC_CONTAINER_BASE_DIR": container_build_dir,
|
||||
"CERC_HOST_UID": f"{os.getuid()}",
|
||||
"CERC_HOST_GID": f"{os.getgid()}",
|
||||
"DOCKER_BUILDKIT": config("DOCKER_BUILDKIT", default="0")
|
||||
"DOCKER_BUILDKIT": config("DOCKER_BUILDKIT", default="0"),
|
||||
}
|
||||
container_build_env.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
|
||||
container_build_env.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {})
|
||||
container_build_env.update({"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {})
|
||||
container_build_env.update(
|
||||
{"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args}
|
||||
if extra_build_args
|
||||
else {}
|
||||
)
|
||||
docker_host_env = os.getenv("DOCKER_HOST")
|
||||
if docker_host_env:
|
||||
container_build_env.update({"DOCKER_HOST": docker_host_env})
|
||||
@@ -67,12 +75,18 @@ def process_container(build_context: BuildContext) -> bool:
|
||||
print(f"Building: {build_context.container}")
|
||||
|
||||
default_container_tag = f"{build_context.container}:local"
|
||||
build_context.container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
|
||||
build_context.container_build_env.update(
|
||||
{"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag}
|
||||
)
|
||||
|
||||
# Check if this is in an external stack
|
||||
if stack_is_external(build_context.stack):
|
||||
container_parent_dir = Path(build_context.stack).parent.parent.joinpath("container-build")
|
||||
temp_build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
|
||||
container_parent_dir = Path(build_context.stack).parent.parent.joinpath(
|
||||
"container-build"
|
||||
)
|
||||
temp_build_dir = container_parent_dir.joinpath(
|
||||
build_context.container.replace("/", "-")
|
||||
)
|
||||
temp_build_script_filename = temp_build_dir.joinpath("build.sh")
|
||||
# Now check if the container exists in the external stack.
|
||||
if not temp_build_script_filename.exists():
|
||||
@@ -90,21 +104,34 @@ def process_container(build_context: BuildContext) -> bool:
|
||||
build_command = build_script_filename.as_posix()
|
||||
else:
|
||||
if opts.o.verbose:
|
||||
print(f"No script file found: {build_script_filename}, using default build script")
|
||||
repo_dir = build_context.container.split('/')[1]
|
||||
# TODO: make this less of a hack -- should be specified in some metadata somewhere
|
||||
# Check if we have a repo for this container. If not, set the context dir to the container-build subdir
|
||||
print(
|
||||
f"No script file found: {build_script_filename}, "
|
||||
"using default build script"
|
||||
)
|
||||
repo_dir = build_context.container.split("/")[1]
|
||||
# TODO: make this less of a hack -- should be specified in
|
||||
# some metadata somewhere. Check if we have a repo for this
|
||||
# container. If not, set the context dir to container-build subdir
|
||||
repo_full_path = os.path.join(build_context.dev_root_path, repo_dir)
|
||||
repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
|
||||
build_command = os.path.join(build_context.container_build_dir,
|
||||
"default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
|
||||
repo_dir_or_build_dir = (
|
||||
repo_full_path if os.path.exists(repo_full_path) else build_dir
|
||||
)
|
||||
build_command = (
|
||||
os.path.join(build_context.container_build_dir, "default-build.sh")
|
||||
+ f" {default_container_tag} {repo_dir_or_build_dir}"
|
||||
)
|
||||
if not opts.o.dry_run:
|
||||
# No PATH at all causes failures with podman.
|
||||
if "PATH" not in build_context.container_build_env:
|
||||
build_context.container_build_env["PATH"] = os.environ["PATH"]
|
||||
if opts.o.verbose:
|
||||
print(f"Executing: {build_command} with environment: {build_context.container_build_env}")
|
||||
build_result = subprocess.run(build_command, shell=True, env=build_context.container_build_env)
|
||||
print(
|
||||
f"Executing: {build_command} with environment: "
|
||||
f"{build_context.container_build_env}"
|
||||
)
|
||||
build_result = subprocess.run(
|
||||
build_command, shell=True, env=build_context.container_build_env
|
||||
)
|
||||
if opts.o.verbose:
|
||||
print(f"Return code is: {build_result.returncode}")
|
||||
if build_result.returncode != 0:
|
||||
@@ -117,33 +144,61 @@ def process_container(build_context: BuildContext) -> bool:
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option('--include', help="only build these containers")
|
||||
@click.option('--exclude', help="don\'t build these containers")
|
||||
@click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
|
||||
@click.option("--include", help="only build these containers")
|
||||
@click.option("--exclude", help="don't build these containers")
|
||||
@click.option(
|
||||
"--force-rebuild",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Override dependency checking -- always rebuild",
|
||||
)
|
||||
@click.option("--extra-build-args", help="Supply extra arguments to build")
|
||||
@click.option("--publish-images", is_flag=True, default=False, help="Publish the built images in the specified image registry")
|
||||
@click.option("--image-registry", help="Specify the image registry for --publish-images")
|
||||
@click.option(
|
||||
"--publish-images",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Publish the built images in the specified image registry",
|
||||
)
|
||||
@click.option(
|
||||
"--image-registry", help="Specify the image registry for --publish-images"
|
||||
)
|
||||
@click.pass_context
|
||||
def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_images, image_registry):
|
||||
'''build the set of containers required for a complete stack'''
|
||||
def command(
|
||||
ctx,
|
||||
include,
|
||||
exclude,
|
||||
force_rebuild,
|
||||
extra_build_args,
|
||||
publish_images,
|
||||
image_registry,
|
||||
):
|
||||
"""build the set of containers required for a complete stack"""
|
||||
|
||||
local_stack = ctx.obj.local_stack
|
||||
stack = ctx.obj.stack
|
||||
|
||||
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
|
||||
container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
|
||||
# See: https://stackoverflow.com/questions/25389095/
|
||||
# python-get-path-of-root-project-structure
|
||||
container_build_dir = (
|
||||
Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
|
||||
)
|
||||
|
||||
if local_stack:
|
||||
dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
|
||||
print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
|
||||
dev_root_path = os.getcwd()[0 : os.getcwd().rindex("stack-orchestrator")]
|
||||
print(
|
||||
f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: "
|
||||
f"{dev_root_path}"
|
||||
)
|
||||
else:
|
||||
dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
|
||||
dev_root_path = os.path.expanduser(
|
||||
config("CERC_REPO_BASE_DIR", default="~/cerc")
|
||||
)
|
||||
|
||||
if not opts.o.quiet:
|
||||
print(f'Dev Root is: {dev_root_path}')
|
||||
print(f"Dev Root is: {dev_root_path}")
|
||||
|
||||
if not os.path.isdir(dev_root_path):
|
||||
print('Dev root directory doesn\'t exist, creating')
|
||||
print("Dev root directory doesn't exist, creating")
|
||||
|
||||
if publish_images:
|
||||
if not image_registry:
|
||||
@@ -151,21 +206,22 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag
|
||||
|
||||
containers_in_scope = get_containers_in_scope(stack)
|
||||
|
||||
container_build_env = make_container_build_env(dev_root_path,
|
||||
container_build_dir,
|
||||
opts.o.debug,
|
||||
force_rebuild,
|
||||
extra_build_args)
|
||||
container_build_env = make_container_build_env(
|
||||
dev_root_path,
|
||||
container_build_dir,
|
||||
opts.o.debug,
|
||||
force_rebuild,
|
||||
extra_build_args,
|
||||
)
|
||||
|
||||
for container in containers_in_scope:
|
||||
if include_exclude_check(container, include, exclude):
|
||||
|
||||
build_context = BuildContext(
|
||||
stack,
|
||||
container,
|
||||
container_build_dir,
|
||||
container_build_env,
|
||||
dev_root_path
|
||||
dev_root_path,
|
||||
)
|
||||
result = process_container(build_context)
|
||||
if result:
|
||||
@@ -174,10 +230,16 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag
|
||||
else:
|
||||
print(f"Error running build for {build_context.container}")
|
||||
if not opts.o.continue_on_error:
|
||||
error_exit("container build failed and --continue-on-error not set, exiting")
|
||||
error_exit(
|
||||
"container build failed and --continue-on-error "
|
||||
"not set, exiting"
|
||||
)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("****** Container Build Error, continuing because --continue-on-error is set")
|
||||
print(
|
||||
"****** Container Build Error, continuing because "
|
||||
"--continue-on-error is set"
|
||||
)
|
||||
else:
|
||||
if opts.o.verbose:
|
||||
print(f"Excluding: {container}")
|
||||
|
||||
@@ -32,14 +32,18 @@ builder_js_image_name = "cerc/builder-js:local"
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option('--include', help="only build these packages")
|
||||
@click.option('--exclude', help="don\'t build these packages")
|
||||
@click.option("--force-rebuild", is_flag=True, default=False,
|
||||
help="Override existing target package version check -- force rebuild")
|
||||
@click.option("--include", help="only build these packages")
|
||||
@click.option("--exclude", help="don't build these packages")
|
||||
@click.option(
|
||||
"--force-rebuild",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Override existing target package version check -- force rebuild",
|
||||
)
|
||||
@click.option("--extra-build-args", help="Supply extra arguments to build")
|
||||
@click.pass_context
|
||||
def command(ctx, include, exclude, force_rebuild, extra_build_args):
|
||||
'''build the set of npm packages required for a complete stack'''
|
||||
"""build the set of npm packages required for a complete stack"""
|
||||
|
||||
quiet = ctx.obj.quiet
|
||||
verbose = ctx.obj.verbose
|
||||
@@ -65,45 +69,54 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
|
||||
sys.exit(1)
|
||||
|
||||
if local_stack:
|
||||
dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
|
||||
print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
|
||||
dev_root_path = os.getcwd()[0 : os.getcwd().rindex("stack-orchestrator")]
|
||||
print(
|
||||
f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: "
|
||||
f"{dev_root_path}"
|
||||
)
|
||||
else:
|
||||
dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
|
||||
dev_root_path = os.path.expanduser(
|
||||
config("CERC_REPO_BASE_DIR", default="~/cerc")
|
||||
)
|
||||
|
||||
build_root_path = os.path.join(dev_root_path, "build-trees")
|
||||
|
||||
if verbose:
|
||||
print(f'Dev Root is: {dev_root_path}')
|
||||
print(f"Dev Root is: {dev_root_path}")
|
||||
|
||||
if not os.path.isdir(dev_root_path):
|
||||
print('Dev root directory doesn\'t exist, creating')
|
||||
print("Dev root directory doesn't exist, creating")
|
||||
os.makedirs(dev_root_path)
|
||||
if not os.path.isdir(dev_root_path):
|
||||
print('Build root directory doesn\'t exist, creating')
|
||||
print("Build root directory doesn't exist, creating")
|
||||
os.makedirs(build_root_path)
|
||||
|
||||
# See: https://stackoverflow.com/a/20885799/1701505
|
||||
from stack_orchestrator import data
|
||||
with importlib.resources.open_text(data, "npm-package-list.txt") as package_list_file:
|
||||
|
||||
with importlib.resources.open_text(
|
||||
data, "npm-package-list.txt"
|
||||
) as package_list_file:
|
||||
all_packages = package_list_file.read().splitlines()
|
||||
|
||||
packages_in_scope = []
|
||||
if stack:
|
||||
stack_config = get_parsed_stack_config(stack)
|
||||
# TODO: syntax check the input here
|
||||
packages_in_scope = stack_config['npms']
|
||||
packages_in_scope = stack_config["npms"]
|
||||
else:
|
||||
packages_in_scope = all_packages
|
||||
|
||||
if verbose:
|
||||
print(f'Packages: {packages_in_scope}')
|
||||
print(f"Packages: {packages_in_scope}")
|
||||
|
||||
def build_package(package):
|
||||
if not quiet:
|
||||
print(f"Building npm package: {package}")
|
||||
repo_dir = package
|
||||
repo_full_path = os.path.join(dev_root_path, repo_dir)
|
||||
# Copy the repo and build that to avoid propagating JS tooling file changes back into the cloned repo
|
||||
# Copy the repo and build that to avoid propagating
|
||||
# JS tooling file changes back into the cloned repo
|
||||
repo_copy_path = os.path.join(build_root_path, repo_dir)
|
||||
# First delete any old build tree
|
||||
if os.path.isdir(repo_copy_path):
|
||||
@@ -116,41 +129,63 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
|
||||
print(f"Copying build tree from: {repo_full_path} to: {repo_copy_path}")
|
||||
if not dry_run:
|
||||
copytree(repo_full_path, repo_copy_path)
|
||||
build_command = ["sh", "-c", f"cd /workspace && build-npm-package-local-dependencies.sh {npm_registry_url}"]
|
||||
build_command = [
|
||||
"sh",
|
||||
"-c",
|
||||
"cd /workspace && "
|
||||
f"build-npm-package-local-dependencies.sh {npm_registry_url}",
|
||||
]
|
||||
if not dry_run:
|
||||
if verbose:
|
||||
print(f"Executing: {build_command}")
|
||||
# Originally we used the PEP 584 merge operator:
|
||||
# envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token} | ({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
|
||||
# but that isn't available in Python 3.8 (default in Ubuntu 20) so for now we use dict.update:
|
||||
envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token,
|
||||
"LACONIC_HOSTED_CONFIG_FILE": "config-hosted.yml" # Convention used by our web app packages
|
||||
}
|
||||
# envs = {"CERC_NPM_AUTH_TOKEN": npm_registry_url_token} |
|
||||
# ({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
|
||||
# but that isn't available in Python 3.8 (default in Ubuntu 20)
|
||||
# so for now we use dict.update:
|
||||
envs = {
|
||||
"CERC_NPM_AUTH_TOKEN": npm_registry_url_token,
|
||||
# Convention used by our web app packages
|
||||
"LACONIC_HOSTED_CONFIG_FILE": "config-hosted.yml",
|
||||
}
|
||||
envs.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
|
||||
envs.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {})
|
||||
envs.update({"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {})
|
||||
envs.update(
|
||||
{"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args}
|
||||
if extra_build_args
|
||||
else {}
|
||||
)
|
||||
try:
|
||||
docker.run(builder_js_image_name,
|
||||
remove=True,
|
||||
interactive=True,
|
||||
tty=True,
|
||||
user=f"{os.getuid()}:{os.getgid()}",
|
||||
envs=envs,
|
||||
# TODO: detect this host name in npm_registry_url rather than hard-wiring it
|
||||
add_hosts=[("gitea.local", "host-gateway")],
|
||||
volumes=[(repo_copy_path, "/workspace")],
|
||||
command=build_command
|
||||
)
|
||||
# Note that although the docs say that build_result should contain
|
||||
# the command output as a string, in reality it is always the empty string.
|
||||
# Since we detect errors via catching exceptions below, we can safely ignore it here.
|
||||
docker.run(
|
||||
builder_js_image_name,
|
||||
remove=True,
|
||||
interactive=True,
|
||||
tty=True,
|
||||
user=f"{os.getuid()}:{os.getgid()}",
|
||||
envs=envs,
|
||||
# TODO: detect this host name in npm_registry_url
|
||||
# rather than hard-wiring it
|
||||
add_hosts=[("gitea.local", "host-gateway")],
|
||||
volumes=[(repo_copy_path, "/workspace")],
|
||||
command=build_command,
|
||||
)
|
||||
# Note that although the docs say that build_result should
|
||||
# contain the command output as a string, in reality it is
|
||||
# always the empty string. Since we detect errors via catching
|
||||
# exceptions below, we can safely ignore it here.
|
||||
except DockerException as e:
|
||||
print(f"Error executing build for {package} in container:\n {e}")
|
||||
if not continue_on_error:
|
||||
print("FATAL Error: build failed and --continue-on-error not set, exiting")
|
||||
print(
|
||||
"FATAL Error: build failed and --continue-on-error "
|
||||
"not set, exiting"
|
||||
)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("****** Build Error, continuing because --continue-on-error is set")
|
||||
print(
|
||||
"****** Build Error, continuing because "
|
||||
"--continue-on-error is set"
|
||||
)
|
||||
|
||||
else:
|
||||
print("Skipped")
|
||||
@@ -168,6 +203,12 @@ def _ensure_prerequisites():
|
||||
# Tell the user how to build it if not
|
||||
images = docker.image.list(builder_js_image_name)
|
||||
if len(images) == 0:
|
||||
print(f"FATAL: builder image: {builder_js_image_name} is required but was not found")
|
||||
print("Please run this command to create it: laconic-so --stack build-support build-containers")
|
||||
print(
|
||||
f"FATAL: builder image: {builder_js_image_name} is required "
|
||||
"but was not found"
|
||||
)
|
||||
print(
|
||||
"Please run this command to create it: "
|
||||
"laconic-so --stack build-support build-containers"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
@@ -24,5 +24,5 @@ class BuildContext:
|
||||
stack: str
|
||||
container: str
|
||||
container_build_dir: Path
|
||||
container_build_env: Mapping[str,str]
|
||||
container_build_env: Mapping[str, str]
|
||||
dev_root_path: str
|
||||
|
||||
@@ -20,21 +20,23 @@ from stack_orchestrator.util import get_parsed_stack_config, warn_exit
|
||||
|
||||
|
||||
def get_containers_in_scope(stack: str):
|
||||
|
||||
containers_in_scope = []
|
||||
if stack:
|
||||
stack_config = get_parsed_stack_config(stack)
|
||||
if "containers" not in stack_config or stack_config["containers"] is None:
|
||||
warn_exit(f"stack {stack} does not define any containers")
|
||||
containers_in_scope = stack_config['containers']
|
||||
containers_in_scope = stack_config["containers"]
|
||||
else:
|
||||
# See: https://stackoverflow.com/a/20885799/1701505
|
||||
from stack_orchestrator import data
|
||||
with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
|
||||
|
||||
with importlib.resources.open_text(
|
||||
data, "container-image-list.txt"
|
||||
) as container_list_file:
|
||||
containers_in_scope = container_list_file.read().splitlines()
|
||||
|
||||
if opts.o.verbose:
|
||||
print(f'Containers: {containers_in_scope}')
|
||||
print(f"Containers: {containers_in_scope}")
|
||||
if stack:
|
||||
print(f"Stack: {stack}")
|
||||
|
||||
|
||||
@@ -18,7 +18,8 @@
|
||||
# env vars:
|
||||
# CERC_REPO_BASE_DIR defaults to ~/cerc
|
||||
|
||||
# TODO: display the available list of containers; allow re-build of either all or specific containers
|
||||
# TODO: display the available list of containers;
|
||||
# allow re-build of either all or specific containers
|
||||
|
||||
import os
|
||||
import sys
|
||||
@@ -32,40 +33,55 @@ from stack_orchestrator.build.build_types import BuildContext
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option('--base-container')
|
||||
@click.option('--source-repo', help="directory containing the webapp to build", required=True)
|
||||
@click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
|
||||
@click.option("--base-container")
|
||||
@click.option(
|
||||
"--source-repo", help="directory containing the webapp to build", required=True
|
||||
)
|
||||
@click.option(
|
||||
"--force-rebuild",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Override dependency checking -- always rebuild",
|
||||
)
|
||||
@click.option("--extra-build-args", help="Supply extra arguments to build")
|
||||
@click.option("--tag", help="Container tag (default: cerc/<app_name>:local)")
|
||||
@click.pass_context
|
||||
def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag):
|
||||
'''build the specified webapp container'''
|
||||
"""build the specified webapp container"""
|
||||
logger = TimedLogger()
|
||||
|
||||
quiet = ctx.obj.quiet
|
||||
debug = ctx.obj.debug
|
||||
verbose = ctx.obj.verbose
|
||||
local_stack = ctx.obj.local_stack
|
||||
stack = ctx.obj.stack
|
||||
|
||||
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
|
||||
container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
|
||||
# See: https://stackoverflow.com/questions/25389095/
|
||||
# python-get-path-of-root-project-structure
|
||||
container_build_dir = (
|
||||
Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
|
||||
)
|
||||
|
||||
if local_stack:
|
||||
dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
|
||||
logger.log(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
|
||||
dev_root_path = os.getcwd()[0 : os.getcwd().rindex("stack-orchestrator")]
|
||||
logger.log(
|
||||
f"Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: "
|
||||
f"{dev_root_path}"
|
||||
)
|
||||
else:
|
||||
dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
|
||||
dev_root_path = os.path.expanduser(
|
||||
config("CERC_REPO_BASE_DIR", default="~/cerc")
|
||||
)
|
||||
|
||||
if verbose:
|
||||
logger.log(f'Dev Root is: {dev_root_path}')
|
||||
logger.log(f"Dev Root is: {dev_root_path}")
|
||||
|
||||
if not base_container:
|
||||
base_container = determine_base_container(source_repo)
|
||||
|
||||
# First build the base container.
|
||||
container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
|
||||
force_rebuild, extra_build_args)
|
||||
container_build_env = build_containers.make_container_build_env(
|
||||
dev_root_path, container_build_dir, debug, force_rebuild, extra_build_args
|
||||
)
|
||||
|
||||
if verbose:
|
||||
logger.log(f"Building base container: {base_container}")
|
||||
@@ -85,12 +101,13 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
|
||||
if verbose:
|
||||
logger.log(f"Base container {base_container} build finished.")
|
||||
|
||||
# Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir.
|
||||
# Now build the target webapp. We use the same build script,
|
||||
# but with a different Dockerfile and work dir.
|
||||
container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
|
||||
container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo)
|
||||
container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir,
|
||||
base_container.replace("/", "-"),
|
||||
"Dockerfile.webapp")
|
||||
container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(
|
||||
container_build_dir, base_container.replace("/", "-"), "Dockerfile.webapp"
|
||||
)
|
||||
if not tag:
|
||||
webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
|
||||
tag = f"cerc/{webapp_name}:local"
|
||||
|
||||
@@ -52,7 +52,8 @@ def _local_tag_for(container: str):
|
||||
|
||||
# See: https://docker-docs.uclv.cu/registry/spec/api/
|
||||
# Emulate this:
|
||||
# $ curl -u "my-username:my-token" -X GET "https://<container-registry-hostname>/v2/cerc-io/cerc/test-container/tags/list"
|
||||
# $ curl -u "my-username:my-token" -X GET \
|
||||
# "https://<container-registry-hostname>/v2/cerc-io/cerc/test-container/tags/list"
|
||||
# {"name":"cerc-io/cerc/test-container","tags":["202402232130","202402232208"]}
|
||||
def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List[str]:
|
||||
# registry looks like: git.vdb.to/cerc-io
|
||||
@@ -60,7 +61,9 @@ def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List
|
||||
url = f"https://{registry_parts[0]}/v2/{registry_parts[1]}/{container}/tags/list"
|
||||
if opts.o.debug:
|
||||
print(f"Fetching tags from: {url}")
|
||||
response = requests.get(url, auth=(registry_info.registry_username, registry_info.registry_token))
|
||||
response = requests.get(
|
||||
url, auth=(registry_info.registry_username, registry_info.registry_token)
|
||||
)
|
||||
if response.status_code == 200:
|
||||
tag_info = response.json()
|
||||
if opts.o.debug:
|
||||
@@ -68,7 +71,10 @@ def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List
|
||||
tags_array = tag_info["tags"]
|
||||
return tags_array
|
||||
else:
|
||||
error_exit(f"failed to fetch tags from image registry, status code: {response.status_code}")
|
||||
error_exit(
|
||||
f"failed to fetch tags from image registry, "
|
||||
f"status code: {response.status_code}"
|
||||
)
|
||||
|
||||
|
||||
def _find_latest(candidate_tags: List[str]):
|
||||
@@ -79,9 +85,9 @@ def _find_latest(candidate_tags: List[str]):
|
||||
return sorted_candidates[-1]
|
||||
|
||||
|
||||
def _filter_for_platform(container: str,
|
||||
registry_info: RegistryInfo,
|
||||
tag_list: List[str]) -> List[str] :
|
||||
def _filter_for_platform(
|
||||
container: str, registry_info: RegistryInfo, tag_list: List[str]
|
||||
) -> List[str]:
|
||||
filtered_tags = []
|
||||
this_machine = platform.machine()
|
||||
# Translate between Python and docker platform names
|
||||
@@ -98,7 +104,7 @@ def _filter_for_platform(container: str,
|
||||
manifest = manifest_cmd.inspect_verbose(remote_tag)
|
||||
if opts.o.debug:
|
||||
print(f"manifest: {manifest}")
|
||||
image_architecture = manifest["Descriptor"]["platform"]["architecture"]
|
||||
image_architecture = manifest["Descriptor"]["platform"]["architecture"]
|
||||
if opts.o.debug:
|
||||
print(f"image_architecture: {image_architecture}")
|
||||
if this_machine == image_architecture:
|
||||
@@ -137,21 +143,44 @@ def _add_local_tag(remote_tag: str, registry: str, local_tag: str):
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option('--include', help="only fetch these containers")
|
||||
@click.option('--exclude', help="don\'t fetch these containers")
|
||||
@click.option("--force-local-overwrite", is_flag=True, default=False, help="Overwrite a locally built image, if present")
|
||||
@click.option("--image-registry", required=True, help="Specify the image registry to fetch from")
|
||||
@click.option("--registry-username", required=True, help="Specify the image registry username")
|
||||
@click.option("--registry-token", required=True, help="Specify the image registry access token")
|
||||
@click.option("--include", help="only fetch these containers")
|
||||
@click.option("--exclude", help="don't fetch these containers")
|
||||
@click.option(
|
||||
"--force-local-overwrite",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Overwrite a locally built image, if present",
|
||||
)
|
||||
@click.option(
|
||||
"--image-registry", required=True, help="Specify the image registry to fetch from"
|
||||
)
|
||||
@click.option(
|
||||
"--registry-username", required=True, help="Specify the image registry username"
|
||||
)
|
||||
@click.option(
|
||||
"--registry-token", required=True, help="Specify the image registry access token"
|
||||
)
|
||||
@click.pass_context
|
||||
def command(ctx, include, exclude, force_local_overwrite, image_registry, registry_username, registry_token):
|
||||
'''EXPERIMENTAL: fetch the images for a stack from remote registry'''
|
||||
def command(
|
||||
ctx,
|
||||
include,
|
||||
exclude,
|
||||
force_local_overwrite,
|
||||
image_registry,
|
||||
registry_username,
|
||||
registry_token,
|
||||
):
|
||||
"""EXPERIMENTAL: fetch the images for a stack from remote registry"""
|
||||
|
||||
registry_info = RegistryInfo(image_registry, registry_username, registry_token)
|
||||
docker = DockerClient()
|
||||
if not opts.o.quiet:
|
||||
print("Logging into container registry:")
|
||||
docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token)
|
||||
docker.login(
|
||||
registry_info.registry,
|
||||
registry_info.registry_username,
|
||||
registry_info.registry_token,
|
||||
)
|
||||
# Generate list of target containers
|
||||
stack = ctx.obj.stack
|
||||
containers_in_scope = get_containers_in_scope(stack)
|
||||
@@ -172,19 +201,24 @@ def command(ctx, include, exclude, force_local_overwrite, image_registry, regist
|
||||
print(f"Fetching: {image_to_fetch}")
|
||||
_fetch_image(image_to_fetch, registry_info)
|
||||
# Now check if the target container already exists exists locally already
|
||||
if (_exists_locally(container)):
|
||||
if _exists_locally(container):
|
||||
if not opts.o.quiet:
|
||||
print(f"Container image {container} already exists locally")
|
||||
# if so, fail unless the user specified force-local-overwrite
|
||||
if (force_local_overwrite):
|
||||
if force_local_overwrite:
|
||||
# In that case remove the existing :local tag
|
||||
if not opts.o.quiet:
|
||||
print(f"Warning: overwriting local tag from this image: {container} because "
|
||||
"--force-local-overwrite was specified")
|
||||
print(
|
||||
f"Warning: overwriting local tag from this image: "
|
||||
f"{container} because --force-local-overwrite was specified"
|
||||
)
|
||||
else:
|
||||
if not opts.o.quiet:
|
||||
print(f"Skipping local tagging for this image: {container} because that would "
|
||||
"overwrite an existing :local tagged image, use --force-local-overwrite to do so.")
|
||||
print(
|
||||
f"Skipping local tagging for this image: {container} "
|
||||
"because that would overwrite an existing :local tagged "
|
||||
"image, use --force-local-overwrite to do so."
|
||||
)
|
||||
continue
|
||||
# Tag the fetched image with the :local tag
|
||||
_add_local_tag(image_to_fetch, image_registry, local_tag)
|
||||
@@ -192,4 +226,7 @@ def command(ctx, include, exclude, force_local_overwrite, image_registry, regist
|
||||
if opts.o.verbose:
|
||||
print(f"Excluding: {container}")
|
||||
if not all_containers_found:
|
||||
print("Warning: couldn't find usable images for one or more containers, this stack will not deploy")
|
||||
print(
|
||||
"Warning: couldn't find usable images for one or more containers, "
|
||||
"this stack will not deploy"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user