forked from cerc-io/stack-orchestrator
Add the ability to filter deployment requests by tag. (#730)
Reviewed-on: cerc-io/stack-orchestrator#730
This commit is contained in:
parent
937b983ec9
commit
88a0236ca9
@ -19,6 +19,8 @@ import shlex
|
|||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
|
||||||
import click
|
import click
|
||||||
|
|
||||||
@ -39,8 +41,19 @@ def process_app_deployment_request(
|
|||||||
dns_suffix,
|
dns_suffix,
|
||||||
deployment_parent_dir,
|
deployment_parent_dir,
|
||||||
kube_config,
|
kube_config,
|
||||||
image_registry
|
image_registry,
|
||||||
|
log_parent_dir
|
||||||
):
|
):
|
||||||
|
run_id = f"{app_deployment_request.id}-{str(time.time()).split('.')[0]}-{str(uuid.uuid4()).split('-')[0]}"
|
||||||
|
log_file = None
|
||||||
|
if log_parent_dir:
|
||||||
|
log_dir = os.path.join(log_parent_dir, app_deployment_request.id)
|
||||||
|
if not os.path.exists(log_dir):
|
||||||
|
os.mkdir(log_dir)
|
||||||
|
log_file_path = os.path.join(log_dir, f"{run_id}.log")
|
||||||
|
print(f"Directing build logs to: {log_file_path}")
|
||||||
|
log_file = open(log_file_path, "wt")
|
||||||
|
|
||||||
# 1. look up application
|
# 1. look up application
|
||||||
app = laconic.get_record(app_deployment_request.attributes.application, require=True)
|
app = laconic.get_record(app_deployment_request.attributes.application, require=True)
|
||||||
|
|
||||||
@ -102,8 +115,10 @@ def process_app_deployment_request(
|
|||||||
needs_k8s_deploy = False
|
needs_k8s_deploy = False
|
||||||
# 6. build container (if needed)
|
# 6. build container (if needed)
|
||||||
if not deployment_record or deployment_record.attributes.application != app.id:
|
if not deployment_record or deployment_record.attributes.application != app.id:
|
||||||
build_container_image(app, deployment_container_tag)
|
# TODO: pull from request
|
||||||
push_container_image(deployment_dir)
|
extra_build_args = []
|
||||||
|
build_container_image(app, deployment_container_tag, extra_build_args, log_file)
|
||||||
|
push_container_image(deployment_dir, log_file)
|
||||||
needs_k8s_deploy = True
|
needs_k8s_deploy = True
|
||||||
|
|
||||||
# 7. update config (if needed)
|
# 7. update config (if needed)
|
||||||
@ -116,6 +131,7 @@ def process_app_deployment_request(
|
|||||||
deploy_to_k8s(
|
deploy_to_k8s(
|
||||||
deployment_record,
|
deployment_record,
|
||||||
deployment_dir,
|
deployment_dir,
|
||||||
|
log_file
|
||||||
)
|
)
|
||||||
|
|
||||||
publish_deployment(
|
publish_deployment(
|
||||||
@ -162,10 +178,14 @@ def dump_known_requests(filename, requests, status="SEEN"):
|
|||||||
@click.option("--record-namespace-dns", help="eg, crn://laconic/dns")
|
@click.option("--record-namespace-dns", help="eg, crn://laconic/dns")
|
||||||
@click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments")
|
@click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments")
|
||||||
@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
|
@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
|
||||||
|
@click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="")
|
||||||
|
@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="")
|
||||||
|
@click.option("--log-dir", help="Output build/deployment logs to directory.", default=None)
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir,
|
def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, # noqa: C901
|
||||||
request_id, discover, state_file, only_update_state,
|
request_id, discover, state_file, only_update_state,
|
||||||
dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run):
|
dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run,
|
||||||
|
include_tags, exclude_tags, log_dir):
|
||||||
if request_id and discover:
|
if request_id and discover:
|
||||||
print("Cannot specify both --request-id and --discover", file=sys.stderr)
|
print("Cannot specify both --request-id and --discover", file=sys.stderr)
|
||||||
sys.exit(2)
|
sys.exit(2)
|
||||||
@ -183,6 +203,10 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
|
|||||||
print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr)
|
print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr)
|
||||||
sys.exit(2)
|
sys.exit(2)
|
||||||
|
|
||||||
|
# Split CSV and clean up values.
|
||||||
|
include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
|
||||||
|
exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
|
||||||
|
|
||||||
laconic = LaconicRegistryClient(laconic_config)
|
laconic = LaconicRegistryClient(laconic_config)
|
||||||
|
|
||||||
# Find deployment requests.
|
# Find deployment requests.
|
||||||
@ -198,12 +222,24 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
|
|||||||
dump_known_requests(state_file, requests)
|
dump_known_requests(state_file, requests)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
def skip_by_tag(r):
|
||||||
|
for tag in exclude_tags:
|
||||||
|
if tag and r.attributes.tags and tag in r.attributes.tags:
|
||||||
|
return True
|
||||||
|
|
||||||
|
for tag in include_tags:
|
||||||
|
if tag and (not r.attributes.tags or tag not in r.attributes.tags):
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
previous_requests = load_known_requests(state_file)
|
previous_requests = load_known_requests(state_file)
|
||||||
|
|
||||||
# Collapse related requests.
|
# Collapse related requests.
|
||||||
requests.sort(key=lambda r: r.createTime)
|
requests.sort(key=lambda r: r.createTime)
|
||||||
requests.reverse()
|
requests.reverse()
|
||||||
requests_by_name = {}
|
requests_by_name = {}
|
||||||
|
skipped_by_name = {}
|
||||||
for r in requests:
|
for r in requests:
|
||||||
# TODO: Do this _after_ filtering deployments and cancellations to minimize round trips.
|
# TODO: Do this _after_ filtering deployments and cancellations to minimize round trips.
|
||||||
app = laconic.get_record(r.attributes.application)
|
app = laconic.get_record(r.attributes.application)
|
||||||
@ -216,17 +252,20 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
|
|||||||
requested_name = generate_hostname_for_app(app)
|
requested_name = generate_hostname_for_app(app)
|
||||||
print("Generating name %s for request %s." % (requested_name, r.id))
|
print("Generating name %s for request %s." % (requested_name, r.id))
|
||||||
|
|
||||||
if requested_name not in requests_by_name:
|
if requested_name in skipped_by_name or requested_name in requests_by_name:
|
||||||
print(
|
print("Ignoring request %s, it has been superseded." % r.id)
|
||||||
"Found request %s to run application %s on %s."
|
continue
|
||||||
% (r.id, r.attributes.application, requested_name)
|
|
||||||
)
|
if skip_by_tag(r):
|
||||||
requests_by_name[requested_name] = r
|
print("Skipping request %s, filtered by tag (include %s, exclude %s, present %s)" % (r.id,
|
||||||
else:
|
include_tags,
|
||||||
print(
|
exclude_tags,
|
||||||
"Ignoring request %s, it is superseded by %s."
|
r.attributes.tags))
|
||||||
% (r.id, requests_by_name[requested_name].id)
|
skipped_by_name[requested_name] = r
|
||||||
)
|
continue
|
||||||
|
|
||||||
|
print("Found request %s to run application %s on %s." % (r.id, r.attributes.application, requested_name))
|
||||||
|
requests_by_name[requested_name] = r
|
||||||
|
|
||||||
# Find deployments.
|
# Find deployments.
|
||||||
deployments = laconic.app_deployments()
|
deployments = laconic.app_deployments()
|
||||||
@ -273,7 +312,8 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
|
|||||||
dns_suffix,
|
dns_suffix,
|
||||||
os.path.abspath(deployment_parent_dir),
|
os.path.abspath(deployment_parent_dir),
|
||||||
kube_config,
|
kube_config,
|
||||||
image_registry
|
image_registry,
|
||||||
|
log_dir
|
||||||
)
|
)
|
||||||
status = "DEPLOYED"
|
status = "DEPLOYED"
|
||||||
finally:
|
finally:
|
||||||
|
@ -212,7 +212,7 @@ def determine_base_container(clone_dir, app_type="webapp"):
|
|||||||
return base_container
|
return base_container
|
||||||
|
|
||||||
|
|
||||||
def build_container_image(app_record, tag, extra_build_args=[]):
|
def build_container_image(app_record, tag, extra_build_args=[], log_file=None):
|
||||||
tmpdir = tempfile.mkdtemp()
|
tmpdir = tempfile.mkdtemp()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@ -227,10 +227,10 @@ def build_container_image(app_record, tag, extra_build_args=[]):
|
|||||||
git_env = dict(os.environ.copy())
|
git_env = dict(os.environ.copy())
|
||||||
# Never prompt
|
# Never prompt
|
||||||
git_env["GIT_TERMINAL_PROMPT"] = "0"
|
git_env["GIT_TERMINAL_PROMPT"] = "0"
|
||||||
subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env)
|
subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env, stdout=log_file, stderr=log_file)
|
||||||
subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env)
|
subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env, stdout=log_file, stderr=log_file)
|
||||||
else:
|
else:
|
||||||
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir])
|
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir], stdout=log_file, stderr=log_file)
|
||||||
result.check_returncode()
|
result.check_returncode()
|
||||||
|
|
||||||
base_container = determine_base_container(clone_dir, app_record.attributes.app_type)
|
base_container = determine_base_container(clone_dir, app_record.attributes.app_type)
|
||||||
@ -246,25 +246,27 @@ def build_container_image(app_record, tag, extra_build_args=[]):
|
|||||||
build_command.append("--extra-build-args")
|
build_command.append("--extra-build-args")
|
||||||
build_command.append(" ".join(extra_build_args))
|
build_command.append(" ".join(extra_build_args))
|
||||||
|
|
||||||
result = subprocess.run(build_command)
|
result = subprocess.run(build_command, stdout=log_file, stderr=log_file)
|
||||||
result.check_returncode()
|
result.check_returncode()
|
||||||
finally:
|
finally:
|
||||||
cmd("rm", "-rf", tmpdir)
|
cmd("rm", "-rf", tmpdir)
|
||||||
|
|
||||||
|
|
||||||
def push_container_image(deployment_dir):
|
def push_container_image(deployment_dir, log_file=None):
|
||||||
print("Pushing image ...")
|
print("Pushing image ...")
|
||||||
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"])
|
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"],
|
||||||
|
stdout=log_file, stderr=log_file)
|
||||||
result.check_returncode()
|
result.check_returncode()
|
||||||
|
|
||||||
|
|
||||||
def deploy_to_k8s(deploy_record, deployment_dir):
|
def deploy_to_k8s(deploy_record, deployment_dir, log_file=None):
|
||||||
if not deploy_record:
|
if not deploy_record:
|
||||||
command = "up"
|
command = "up"
|
||||||
else:
|
else:
|
||||||
command = "update"
|
command = "update"
|
||||||
|
|
||||||
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command])
|
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command],
|
||||||
|
stdout=log_file, stderr=log_file)
|
||||||
result.check_returncode()
|
result.check_returncode()
|
||||||
|
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user