cerc-io/stack-orchestrator
17
1
Fork 3
Code Issues 113 Pull Requests 19 Actions Packages Projects Releases 480 Wiki Activity

Refactor build code

Browse Source
This commit is contained in:
David Boreham 2024-02-23 09:51:18 -07:00
parent 2af6ffce77
commit f37d33afbe
2 changed files with 71 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
stack_orchestrator/build/build_containers.py
View File

@ -27,8 +27,10 @@ import subprocess
import click import click
import importlib.resources import importlib.resources
from pathlib import Path from pathlib import Path
from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, warn_exit from stack_orchestrator.opts import opts
from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, error_exit, warn_exit
from stack_orchestrator.base import get_npm_registry_url from stack_orchestrator.base import get_npm_registry_url
from stack_orchestrator.build.build_types import BuildContext
# TODO: find a place for this # TODO: find a place for this
# epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)" # epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
@ -59,63 +61,54 @@ def make_container_build_env(dev_root_path: str,
return container_build_env return container_build_env
def process_container(stack: str, def process_container(build_context: BuildContext):
container, if not opts.o.quiet:
container_build_dir: str, print(f"Building: {build_context.container}")
container_build_env: dict,
dev_root_path: str,
quiet: bool,
verbose: bool,
dry_run: bool,
continue_on_error: bool,
):
if not quiet:
print(f"Building: {container}")
default_container_tag = f"{container}:local" default_container_tag = f"{build_context.container}:local"
container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag}) build_context.container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
# Check if this is in an external stack # Check if this is in an external stack
if stack_is_external(stack): if stack_is_external(build_context.stack):
container_parent_dir = Path(stack).joinpath("container-build") container_parent_dir = Path(build_context.stack).joinpath("container-build")
temp_build_dir = container_parent_dir.joinpath(container.replace("/", "-")) temp_build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
temp_build_script_filename = temp_build_dir.joinpath("build.sh") temp_build_script_filename = temp_build_dir.joinpath("build.sh")
# Now check if the container exists in the external stack. # Now check if the container exists in the external stack.
if not temp_build_script_filename.exists(): if not temp_build_script_filename.exists():
# If not, revert to building an internal container # If not, revert to building an internal container
container_parent_dir = container_build_dir container_parent_dir = build_context.container_build_dir
else: else:
container_parent_dir = container_build_dir container_parent_dir = build_context.container_build_dir
build_dir = container_parent_dir.joinpath(container.replace("/", "-")) build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
build_script_filename = build_dir.joinpath("build.sh") build_script_filename = build_dir.joinpath("build.sh")
if verbose: if opts.o.verbose:
print(f"Build script filename: {build_script_filename}") print(f"Build script filename: {build_script_filename}")
if os.path.exists(build_script_filename): if os.path.exists(build_script_filename):
build_command = build_script_filename.as_posix() build_command = build_script_filename.as_posix()
else: else:
if verbose: if opts.o.verbose:
print(f"No script file found: {build_script_filename}, using default build script") print(f"No script file found: {build_script_filename}, using default build script")
repo_dir = container.split('/')[1] repo_dir = build_context.container.split('/')[1]
# TODO: make this less of a hack -- should be specified in some metadata somewhere # TODO: make this less of a hack -- should be specified in some metadata somewhere
# Check if we have a repo for this container. If not, set the context dir to the container-build subdir # Check if we have a repo for this container. If not, set the context dir to the container-build subdir
repo_full_path = os.path.join(dev_root_path, repo_dir) repo_full_path = os.path.join(build_context.dev_root_path, repo_dir)
repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
build_command = os.path.join(container_build_dir, build_command = os.path.join(build_context.container_build_dir,
"default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}" "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
if not dry_run: if not opts.o.dry_run:
# No PATH at all causes failures with podman. # No PATH at all causes failures with podman.
if "PATH" not in container_build_env: if "PATH" not in build_context.container_build_env:
container_build_env["PATH"] = os.environ["PATH"] build_context.container_build_env["PATH"] = os.environ["PATH"]
if verbose: if opts.o.verbose:
print(f"Executing: {build_command} with environment: {container_build_env}") print(f"Executing: {build_command} with environment: {build_context.container_build_env}")
build_result = subprocess.run(build_command, shell=True, env=container_build_env) build_result = subprocess.run(build_command, shell=True, env=build_context.container_build_env)
if verbose: if opts.o.verbose:
print(f"Return code is: {build_result.returncode}") print(f"Return code is: {build_result.returncode}")
if build_result.returncode != 0: if build_result.returncode != 0:
print(f"Error running build for {container}") print(f"Error running build for {build_context.container}")
if not continue_on_error: if not opts.o.continue_on_error:
print("FATAL Error: container build failed and --continue-on-error not set, exiting") print("FATAL Error: container build failed and --continue-on-error not set, exiting")
sys.exit(1) sys.exit(1)
else: else:
@ -129,17 +122,14 @@ def process_container(stack: str,
@click.option('--exclude', help="don\'t build these containers") @click.option('--exclude', help="don\'t build these containers")
@click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild") @click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
@click.option("--extra-build-args", help="Supply extra arguments to build") @click.option("--extra-build-args", help="Supply extra arguments to build")
@click.option("--publish-images", is_flag=True, default=False, help="Publish the built images in the specified image registry")
@click.option("--image-registry-url", help="Specify the image registry for --publish-images")
@click.pass_context @click.pass_context
def command(ctx, include, exclude, force_rebuild, extra_build_args): def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_images, image_registry_url):
'''build the set of containers required for a complete stack''' '''build the set of containers required for a complete stack'''
quiet = ctx.obj.quiet
verbose = ctx.obj.verbose
dry_run = ctx.obj.dry_run
debug = ctx.obj.debug
local_stack = ctx.obj.local_stack local_stack = ctx.obj.local_stack
stack = ctx.obj.stack stack = ctx.obj.stack
continue_on_error = ctx.obj.continue_on_error
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build") container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
@ -150,12 +140,16 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
else: else:
dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
if not quiet: if not opts.o.quiet:
print(f'Dev Root is: {dev_root_path}') print(f'Dev Root is: {dev_root_path}')
if not os.path.isdir(dev_root_path): if not os.path.isdir(dev_root_path):
print('Dev root directory doesn\'t exist, creating') print('Dev root directory doesn\'t exist, creating')
if publish_images:
if not image_registry_url:
error_exit("ERROR: --image-registry-url must be supplied with --publish-images")
# See: https://stackoverflow.com/a/20885799/1701505 # See: https://stackoverflow.com/a/20885799/1701505
from stack_orchestrator import data from stack_orchestrator import data
with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file: with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
@ -170,21 +164,23 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
else: else:
containers_in_scope = all_containers containers_in_scope = all_containers
if verbose: if opts.o.verbose:
print(f'Containers: {containers_in_scope}') print(f'Containers: {containers_in_scope}')
if stack: if stack:
print(f"Stack: {stack}") print(f"Stack: {stack}")
container_build_env = make_container_build_env(dev_root_path, container_build_env = make_container_build_env(dev_root_path,
container_build_dir, container_build_dir,
debug, opts.o.debug,
force_rebuild, opts.o.force_rebuild,
extra_build_args) extra_build_args)
for container in containers_in_scope: for container in containers_in_scope:
if include_exclude_check(container, include, exclude): if include_exclude_check(container, include, exclude):
process_container(stack, container, container_build_dir, container_build_env, build_context = BuildContext(
dev_root_path, quiet, verbose, dry_run, continue_on_error)
)
process_container(build_context)
else: else:
if verbose: if opts.o.verbose:
print(f"Excluding: {container}") print(f"Excluding: {container}")

27
stack_orchestrator/build/build_types.py Normal file
View File

@ -0,0 +1,27 @@
# Copyright © 2024 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from dataclasses import dataclass
from pathlib import Path
from typing import Mapping
@dataclass
class BuildContext:
stack: str
container: str
container_build_dir: Path
container_build_env: Mapping[str,str]