Merge pull request #253 from cerc-io/dboreham/wait-for-export-cluster-config

Detect transient errors exporting variables and re-try

Former-commit-id: a4a9607ea8

app/data/compose/docker-compose-go-ethereum-foundry.yml

@@ -8,7 +8,7 @@ services:
         condition: service_healthy
     image: cerc/go-ethereum-foundry:local
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "8545"]
+      test: ["CMD", "nc", "-vz", "localhost", "8545"]
       interval: 30s
       timeout: 3s
       retries: 10

app/data/compose/docker-compose-watcher-erc20.yml

@@ -39,7 +39,7 @@ services:
       - "0.0.0.0:3002:3001"
       - "0.0.0.0:9002:9001"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3002"]
+      test: ["CMD", "nc", "-vz", "localhost", "3001"]
       interval: 20s
       timeout: 5s
       retries: 15

app/data/container-build/cerc-go-ethereum-foundry/Dockerfile

@@ -1,7 +1,9 @@
+# Note: cerc/foundry is Debian based
 FROM cerc/foundry:local
 
-RUN apk update ; apk add --no-cache --allow-untrusted ca-certificates curl bash git jq
-RUN apk add --no-cache --upgrade grep
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -y install --no-install-recommends jq curl netcat
+
 WORKDIR /root
 
 ARG GENESIS_FILE_PATH=genesis.json

app/data/version.txt

@@ -1,2 +1,2 @@
 # This file should be re-generated running: scripts/update-version-file.sh script
-v1.0.32-1aeb44d
+v1.0.33-7e1137f

app/deploy_system.py

@@ -292,21 +292,41 @@ def _orchestrate_cluster_config(ctx, cluster_config, docker, container_exec_env)
                     directive
                 )
                 if ctx.verbose:
-                    print(f"Setting {pd.destination_container}.{pd.destination_variable} = {pd.source_container}.{pd.source_variable}")
-                # TODO: fix the script paths so they're consistent between containers
-                source_value = docker.compose.execute(pd.source_container,
-                                                      ["sh", "-c",
-                                                          f"sh /docker-entrypoint-scripts.d/export-{pd.source_variable}.sh"],
-                                                      tty=False,
-                                                      envs=container_exec_env)
-                # TODO: handle the case that the value is not yet available
-                if ctx.debug:
-                    print(f"fetched source value: {source_value}")
-                destination_output = docker.compose.execute(pd.destination_container,
-                                                            ["sh", "-c",
-                                                             f"sh /scripts/import-{pd.destination_variable}.sh {source_value}"],
-                                                            tty=False,
-                                                            envs=container_exec_env)
-                if ctx.debug:
-                    print(f"destination output: {destination_output}")
-                # TODO: detect errors here
+                    print(f"Setting {pd.destination_container}.{pd.destination_variable}"
+                          f" = {pd.source_container}.{pd.source_variable}")
+                # TODO: add a timeout
+                waiting_for_data = True
+                while waiting_for_data:
+                    # TODO: fix the script paths so they're consistent between containers
+                    source_value = None
+                    try:
+                        source_value = docker.compose.execute(pd.source_container,
+                                                              ["sh", "-c",
+                                                                  "sh /docker-entrypoint-scripts.d/export-"
+                                                                  f"{pd.source_variable}.sh"],
+                                                              tty=False,
+                                                              envs=container_exec_env)
+                    except DockerException as error:
+                        if ctx.debug:
+                            print(f"Docker exception reading config source: {error}")
+                        # If the script executed failed for some reason, we get:
+                        # "It returned with code 1"
+                        if "It returned with code 1" in str(error):
+                            if ctx.verbose:
+                                print("Config export script returned an error, re-trying")
+                        # If the script failed to execute (e.g. the file is not there) then we get:
+                        # "It returned with code 2"
+                        if "It returned with code 2" in str(error):
+                            print(f"Fatal error reading config source: {error}")
+                    if source_value:
+                        if ctx.debug:
+                            print(f"fetched source value: {source_value}")
+                        destination_output = docker.compose.execute(pd.destination_container,
+                                                                    ["sh", "-c",
+                                                                        f"sh /scripts/import-{pd.destination_variable}.sh"
+                                                                        f" {source_value}"],
+                                                                    tty=False,
+                                                                    envs=container_exec_env)
+                        waiting_for_data = False
+                    if ctx.debug:
+                        print(f"destination output: {destination_output}")