cerc-io/stack-orchestrator
17
1
Fork 3
Code Issues 112 Pull Requests 19 Actions Packages Projects Releases 480 Wiki Activity

Detect transient errors exporting variables and re-try

Browse Source 
Former-commit-id: 15c0a92f30
This commit is contained in:
David Boreham 2023-03-28 11:44:02 -06:00
parent 29f0302fb0
commit 75ca0d4138
1 changed files with 38 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
app/deploy_system.py
View File

@ -292,21 +292,41 @@ def _orchestrate_cluster_config(ctx, cluster_config, docker, container_exec_env)
directive directive
) )
if ctx.verbose: if ctx.verbose:
print(f"Setting {pd.destination_container}.{pd.destination_variable} = {pd.source_container}.{pd.source_variable}") print(f"Setting {pd.destination_container}.{pd.destination_variable}"
# TODO: fix the script paths so they're consistent between containers f" = {pd.source_container}.{pd.source_variable}")
source_value = docker.compose.execute(pd.source_container, # TODO: add a timeout
["sh", "-c", waiting_for_data = True
f"sh /docker-entrypoint-scripts.d/export-{pd.source_variable}.sh"], while waiting_for_data:
tty=False, # TODO: fix the script paths so they're consistent between containers
envs=container_exec_env) source_value = None
# TODO: handle the case that the value is not yet available try:
if ctx.debug: source_value = docker.compose.execute(pd.source_container,
print(f"fetched source value: {source_value}") ["sh", "-c",
destination_output = docker.compose.execute(pd.destination_container, "sh /docker-entrypoint-scripts.d/export-"
["sh", "-c", f"{pd.source_variable}.sh"],
f"sh /scripts/import-{pd.destination_variable}.sh {source_value}"], tty=False,
tty=False, envs=container_exec_env)
envs=container_exec_env) except DockerException as error:
if ctx.debug: if ctx.debug:
print(f"destination output: {destination_output}") print(f"Docker exception reading config source: {error}")
# TODO: detect errors here # If the script executed failed for some reason, we get:
# "It returned with code 1"
if "It returned with code 1" in str(error):
if ctx.verbose:
print("Config export script returned an error, re-trying")
# If the script failed to execute (e.g. the file is not there) then we get:
# "It returned with code 2"
if "It returned with code 2" in str(error):
print(f"Fatal error reading config source: {error}")
if source_value:
if ctx.debug:
print(f"fetched source value: {source_value}")
destination_output = docker.compose.execute(pd.destination_container,
["sh", "-c",
f"sh /scripts/import-{pd.destination_variable}.sh"
f" {source_value}"],
tty=False,
envs=container_exec_env)
waiting_for_data = False
if ctx.debug:
print(f"destination output: {destination_output}")