2023-09-10 19:28:26 +00:00
|
|
|
# Copyright © 2022, 2023 Vulcanize
|
2022-08-23 21:58:32 +00:00
|
|
|
|
|
|
|
# This program is free software: you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
|
|
# (at your option) any later version.
|
|
|
|
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
# GNU Affero General Public License for more details.
|
|
|
|
|
|
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
|
|
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
|
|
|
|
|
2022-08-12 14:54:09 +00:00
|
|
|
# Builds or pulls containers for the system components
|
|
|
|
|
|
|
|
# env vars:
|
2022-08-23 22:02:38 +00:00
|
|
|
# CERC_REPO_BASE_DIR defaults to ~/cerc
|
2022-08-12 14:54:09 +00:00
|
|
|
|
|
|
|
# TODO: display the available list of containers; allow re-build of either all or specific containers
|
|
|
|
|
|
|
|
import os
|
2023-01-19 20:20:53 +00:00
|
|
|
import sys
|
2022-08-12 14:54:09 +00:00
|
|
|
from decouple import config
|
|
|
|
import subprocess
|
2022-08-24 03:27:42 +00:00
|
|
|
import click
|
2023-01-09 03:08:35 +00:00
|
|
|
from pathlib import Path
|
2024-02-23 19:57:47 +00:00
|
|
|
from stack_orchestrator.opts import opts
|
2024-02-27 15:15:08 +00:00
|
|
|
from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
|
2023-11-07 07:06:55 +00:00
|
|
|
from stack_orchestrator.base import get_npm_registry_url
|
2024-02-23 19:57:47 +00:00
|
|
|
from stack_orchestrator.build.build_types import BuildContext
|
|
|
|
from stack_orchestrator.build.publish import publish_image
|
2024-02-27 15:15:08 +00:00
|
|
|
from stack_orchestrator.build.build_util import get_containers_in_scope
|
2022-08-12 14:54:09 +00:00
|
|
|
|
2022-08-24 18:53:51 +00:00
|
|
|
# TODO: find a place for this
|
2022-08-24 18:36:58 +00:00
|
|
|
# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
|
2022-08-12 14:54:09 +00:00
|
|
|
|
2024-01-31 05:09:48 +00:00
|
|
|
|
2023-11-08 00:15:04 +00:00
|
|
|
def make_container_build_env(dev_root_path: str,
|
|
|
|
container_build_dir: str,
|
|
|
|
debug: bool,
|
|
|
|
force_rebuild: bool,
|
|
|
|
extra_build_args: str):
|
|
|
|
container_build_env = {
|
|
|
|
"CERC_NPM_REGISTRY_URL": get_npm_registry_url(),
|
|
|
|
"CERC_GO_AUTH_TOKEN": config("CERC_GO_AUTH_TOKEN", default=""),
|
|
|
|
"CERC_NPM_AUTH_TOKEN": config("CERC_NPM_AUTH_TOKEN", default=""),
|
|
|
|
"CERC_REPO_BASE_DIR": dev_root_path,
|
|
|
|
"CERC_CONTAINER_BASE_DIR": container_build_dir,
|
|
|
|
"CERC_HOST_UID": f"{os.getuid()}",
|
|
|
|
"CERC_HOST_GID": f"{os.getgid()}",
|
|
|
|
"DOCKER_BUILDKIT": config("DOCKER_BUILDKIT", default="0")
|
|
|
|
}
|
|
|
|
container_build_env.update({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
|
|
|
|
container_build_env.update({"CERC_FORCE_REBUILD": "true"} if force_rebuild else {})
|
|
|
|
container_build_env.update({"CERC_CONTAINER_EXTRA_BUILD_ARGS": extra_build_args} if extra_build_args else {})
|
|
|
|
docker_host_env = os.getenv("DOCKER_HOST")
|
|
|
|
if docker_host_env:
|
|
|
|
container_build_env.update({"DOCKER_HOST": docker_host_env})
|
|
|
|
|
|
|
|
return container_build_env
|
|
|
|
|
|
|
|
|
2024-02-23 19:57:47 +00:00
|
|
|
def process_container(build_context: BuildContext) -> bool:
|
|
|
|
if not opts.o.quiet:
|
|
|
|
print(f"Building: {build_context.container}")
|
2023-11-15 03:59:48 +00:00
|
|
|
|
2024-02-23 19:57:47 +00:00
|
|
|
default_container_tag = f"{build_context.container}:local"
|
|
|
|
build_context.container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
|
2023-11-15 03:59:48 +00:00
|
|
|
|
|
|
|
# Check if this is in an external stack
|
2024-02-23 19:57:47 +00:00
|
|
|
if stack_is_external(build_context.stack):
|
|
|
|
container_parent_dir = Path(build_context.stack).joinpath("container-build")
|
|
|
|
temp_build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
|
2023-11-15 03:59:48 +00:00
|
|
|
temp_build_script_filename = temp_build_dir.joinpath("build.sh")
|
|
|
|
# Now check if the container exists in the external stack.
|
|
|
|
if not temp_build_script_filename.exists():
|
|
|
|
# If not, revert to building an internal container
|
2024-02-23 19:57:47 +00:00
|
|
|
container_parent_dir = build_context.container_build_dir
|
2023-11-15 03:59:48 +00:00
|
|
|
else:
|
2024-02-23 19:57:47 +00:00
|
|
|
container_parent_dir = build_context.container_build_dir
|
2023-11-15 03:59:48 +00:00
|
|
|
|
2024-02-23 19:57:47 +00:00
|
|
|
build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
|
2023-11-15 03:59:48 +00:00
|
|
|
build_script_filename = build_dir.joinpath("build.sh")
|
|
|
|
|
2024-02-23 19:57:47 +00:00
|
|
|
if opts.o.verbose:
|
2023-11-08 00:15:04 +00:00
|
|
|
print(f"Build script filename: {build_script_filename}")
|
|
|
|
if os.path.exists(build_script_filename):
|
2023-11-15 03:59:48 +00:00
|
|
|
build_command = build_script_filename.as_posix()
|
2023-11-08 00:15:04 +00:00
|
|
|
else:
|
2024-02-23 19:57:47 +00:00
|
|
|
if opts.o.verbose:
|
2023-11-08 00:15:04 +00:00
|
|
|
print(f"No script file found: {build_script_filename}, using default build script")
|
2024-02-23 19:57:47 +00:00
|
|
|
repo_dir = build_context.container.split('/')[1]
|
2023-11-08 00:15:04 +00:00
|
|
|
# TODO: make this less of a hack -- should be specified in some metadata somewhere
|
|
|
|
# Check if we have a repo for this container. If not, set the context dir to the container-build subdir
|
2024-02-23 19:57:47 +00:00
|
|
|
repo_full_path = os.path.join(build_context.dev_root_path, repo_dir)
|
2023-11-08 00:15:04 +00:00
|
|
|
repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
|
2024-02-23 19:57:47 +00:00
|
|
|
build_command = os.path.join(build_context.container_build_dir,
|
2023-11-15 03:59:48 +00:00
|
|
|
"default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
|
2024-02-23 19:57:47 +00:00
|
|
|
if not opts.o.dry_run:
|
2024-01-31 05:09:48 +00:00
|
|
|
# No PATH at all causes failures with podman.
|
2024-02-23 19:57:47 +00:00
|
|
|
if "PATH" not in build_context.container_build_env:
|
|
|
|
build_context.container_build_env["PATH"] = os.environ["PATH"]
|
|
|
|
if opts.o.verbose:
|
|
|
|
print(f"Executing: {build_command} with environment: {build_context.container_build_env}")
|
|
|
|
build_result = subprocess.run(build_command, shell=True, env=build_context.container_build_env)
|
|
|
|
if opts.o.verbose:
|
2023-11-08 00:15:04 +00:00
|
|
|
print(f"Return code is: {build_result.returncode}")
|
|
|
|
if build_result.returncode != 0:
|
2024-02-23 19:57:47 +00:00
|
|
|
return False
|
|
|
|
else:
|
|
|
|
return True
|
2023-11-08 00:15:04 +00:00
|
|
|
else:
|
|
|
|
print("Skipped")
|
2024-02-23 19:57:47 +00:00
|
|
|
return True
|
2022-10-04 01:37:18 +00:00
|
|
|
|
2024-01-31 05:09:48 +00:00
|
|
|
|
2022-08-24 03:27:42 +00:00
|
|
|
@click.command()
|
2022-09-28 16:07:13 +00:00
|
|
|
@click.option('--include', help="only build these containers")
|
|
|
|
@click.option('--exclude', help="don\'t build these containers")
|
2023-04-14 20:19:27 +00:00
|
|
|
@click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
|
|
|
|
@click.option("--extra-build-args", help="Supply extra arguments to build")
|
2024-02-23 19:57:47 +00:00
|
|
|
@click.option("--publish-images", is_flag=True, default=False, help="Publish the built images in the specified image registry")
|
|
|
|
@click.option("--image-registry", help="Specify the image registry for --publish-images")
|
2022-08-24 18:36:58 +00:00
|
|
|
@click.pass_context
|
2024-02-23 19:57:47 +00:00
|
|
|
def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_images, image_registry):
|
2022-08-24 18:53:51 +00:00
|
|
|
'''build the set of containers required for a complete stack'''
|
2022-08-24 18:36:58 +00:00
|
|
|
|
2022-09-29 18:46:21 +00:00
|
|
|
local_stack = ctx.obj.local_stack
|
2023-01-18 22:27:52 +00:00
|
|
|
stack = ctx.obj.stack
|
2022-08-12 14:54:09 +00:00
|
|
|
|
2023-01-09 03:08:35 +00:00
|
|
|
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
|
2023-10-25 04:13:09 +00:00
|
|
|
container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
|
2022-12-07 18:38:32 +00:00
|
|
|
|
2022-09-29 18:46:21 +00:00
|
|
|
if local_stack:
|
2022-10-06 12:50:42 +00:00
|
|
|
dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
|
2022-09-29 18:46:21 +00:00
|
|
|
print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
|
|
|
|
else:
|
|
|
|
dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
|
2022-08-12 14:54:09 +00:00
|
|
|
|
2024-02-23 19:57:47 +00:00
|
|
|
if not opts.o.quiet:
|
2022-08-24 03:27:42 +00:00
|
|
|
print(f'Dev Root is: {dev_root_path}')
|
2022-08-12 14:54:09 +00:00
|
|
|
|
2022-08-24 03:27:42 +00:00
|
|
|
if not os.path.isdir(dev_root_path):
|
2022-10-04 01:37:18 +00:00
|
|
|
print('Dev root directory doesn\'t exist, creating')
|
2022-08-12 14:54:09 +00:00
|
|
|
|
2024-02-23 19:57:47 +00:00
|
|
|
if publish_images:
|
|
|
|
if not image_registry:
|
|
|
|
error_exit("--image-registry must be supplied with --publish-images")
|
|
|
|
|
2024-02-27 15:15:08 +00:00
|
|
|
containers_in_scope = get_containers_in_scope(stack)
|
2022-08-24 03:27:42 +00:00
|
|
|
|
2023-11-08 00:15:04 +00:00
|
|
|
container_build_env = make_container_build_env(dev_root_path,
|
|
|
|
container_build_dir,
|
2024-02-23 19:57:47 +00:00
|
|
|
opts.o.debug,
|
2023-11-08 00:15:04 +00:00
|
|
|
force_rebuild,
|
|
|
|
extra_build_args)
|
2022-08-24 03:27:42 +00:00
|
|
|
|
2023-01-18 22:27:52 +00:00
|
|
|
for container in containers_in_scope:
|
2022-09-28 16:07:13 +00:00
|
|
|
if include_exclude_check(container, include, exclude):
|
2024-02-23 19:57:47 +00:00
|
|
|
|
|
|
|
build_context = BuildContext(
|
|
|
|
stack,
|
|
|
|
container,
|
|
|
|
container_build_dir,
|
|
|
|
container_build_env,
|
|
|
|
dev_root_path
|
|
|
|
)
|
|
|
|
result = process_container(build_context)
|
|
|
|
if result:
|
|
|
|
if publish_images:
|
2024-02-23 20:15:37 +00:00
|
|
|
publish_image(f"{container}:local", image_registry)
|
2024-02-23 19:57:47 +00:00
|
|
|
else:
|
|
|
|
print(f"Error running build for {build_context.container}")
|
|
|
|
if not opts.o.continue_on_error:
|
|
|
|
error_exit("container build failed and --continue-on-error not set, exiting")
|
|
|
|
sys.exit(1)
|
|
|
|
else:
|
|
|
|
print("****** Container Build Error, continuing because --continue-on-error is set")
|
2022-09-28 16:07:13 +00:00
|
|
|
else:
|
2024-02-23 19:57:47 +00:00
|
|
|
if opts.o.verbose:
|
2022-09-28 16:07:13 +00:00
|
|
|
print(f"Excluding: {container}")
|