From 643e41d91e95d728ffa051ebb73021d9bc210cb7 Mon Sep 17 00:00:00 2001
From: David Boreham <david@bozemanpass.com>
Date: Tue, 6 Feb 2024 13:48:38 -0700
Subject: [PATCH 1/9] Test database stack

---
 .../compose/docker-compose-test-database.yml     | 16 ++++++++++++++++
 .../cerc-test-database-container/Dockerfile      |  3 +++
 .../cerc-test-database-container/build.sh        |  5 +++++
 .../data/stacks/test-database/README.md          |  3 +++
 .../data/stacks/test-database/stack.yml          |  8 ++++++++
 stack_orchestrator/repos/setup_repositories.py   |  6 +++---
 stack_orchestrator/util.py                       |  5 +++++
 7 files changed, 43 insertions(+), 3 deletions(-)
 create mode 100644 stack_orchestrator/data/compose/docker-compose-test-database.yml
 create mode 100644 stack_orchestrator/data/container-build/cerc-test-database-container/Dockerfile
 create mode 100755 stack_orchestrator/data/container-build/cerc-test-database-container/build.sh
 create mode 100644 stack_orchestrator/data/stacks/test-database/README.md
 create mode 100644 stack_orchestrator/data/stacks/test-database/stack.yml

diff --git a/stack_orchestrator/data/compose/docker-compose-test-database.yml b/stack_orchestrator/data/compose/docker-compose-test-database.yml
new file mode 100644
index 00000000..7de4712b
--- /dev/null
+++ b/stack_orchestrator/data/compose/docker-compose-test-database.yml
@@ -0,0 +1,16 @@
+services:
+  database:
+    image: cerc/test-database-container:local
+    restart: always
+    volumes:
+      - db-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: "test-user"
+      POSTGRES_DB: "test-db"
+      POSTGRES_PASSWORD: "password"
+      POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C"
+    ports:
+      - "5432"
+
+volumes:
+  db-data:
diff --git a/stack_orchestrator/data/container-build/cerc-test-database-container/Dockerfile b/stack_orchestrator/data/container-build/cerc-test-database-container/Dockerfile
new file mode 100644
index 00000000..aae60175
--- /dev/null
+++ b/stack_orchestrator/data/container-build/cerc-test-database-container/Dockerfile
@@ -0,0 +1,3 @@
+FROM postgres:16-bullseye
+
+EXPOSE 5432
diff --git a/stack_orchestrator/data/container-build/cerc-test-database-container/build.sh b/stack_orchestrator/data/container-build/cerc-test-database-container/build.sh
new file mode 100755
index 00000000..a4515229
--- /dev/null
+++ b/stack_orchestrator/data/container-build/cerc-test-database-container/build.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Build cerc/test-container
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+docker build -t cerc/test-database-container:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR
diff --git a/stack_orchestrator/data/stacks/test-database/README.md b/stack_orchestrator/data/stacks/test-database/README.md
new file mode 100644
index 00000000..1dcdcc7b
--- /dev/null
+++ b/stack_orchestrator/data/stacks/test-database/README.md
@@ -0,0 +1,3 @@
+# Test Database Stack
+
+A stack with a database for test/demo purposes.
\ No newline at end of file
diff --git a/stack_orchestrator/data/stacks/test-database/stack.yml b/stack_orchestrator/data/stacks/test-database/stack.yml
new file mode 100644
index 00000000..6fac54e0
--- /dev/null
+++ b/stack_orchestrator/data/stacks/test-database/stack.yml
@@ -0,0 +1,8 @@
+version: "1.0"
+name: test
+description: "A test database stack"
+repos:
+containers:
+  - cerc/test-database-container
+pods:
+  - test-database
diff --git a/stack_orchestrator/repos/setup_repositories.py b/stack_orchestrator/repos/setup_repositories.py
index 3612aed0..a137d645 100644
--- a/stack_orchestrator/repos/setup_repositories.py
+++ b/stack_orchestrator/repos/setup_repositories.py
@@ -26,7 +26,7 @@ import importlib.resources
 from pathlib import Path
 import yaml
 from stack_orchestrator.constants import stack_file_name
-from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
+from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit, warn_exit
 
 
 class GitProgress(git.RemoteProgress):
@@ -249,8 +249,8 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches
             error_exit(f"stack {stack} does not exist")
         with stack_file_path:
             stack_config = yaml.safe_load(open(stack_file_path, "r"))
-            if "repos" not in stack_config:
-                error_exit(f"stack {stack} does not define any repositories")
+            if "repos" not in stack_config or stack_config["repos"] is None:
+                warn_exit(f"stack {stack} does not define any repositories")
             else:
                 repos_in_scope = stack_config["repos"]
     else:
diff --git a/stack_orchestrator/util.py b/stack_orchestrator/util.py
index 36c6bfd0..257e1deb 100644
--- a/stack_orchestrator/util.py
+++ b/stack_orchestrator/util.py
@@ -189,5 +189,10 @@ def error_exit(s):
     sys.exit(1)
 
 
+def warn_exit(s):
+    print(f"WARN: {s}")
+    sys.exit(0)
+
+
 def env_var_map_from_file(file: Path) -> Mapping[str, str]:
     return dotenv_values(file)
-- 
2.45.2


From 4a116d04c11a1cf773a1c9a036a386c4ec07941e Mon Sep 17 00:00:00 2001
From: David Boreham <david@bozemanpass.com>
Date: Thu, 8 Feb 2024 13:29:49 -0700
Subject: [PATCH 2/9] Add test file

---
 tests/database/run-test.sh | 112 +++++++++++++++++++++++++++++++++++++
 1 file changed, 112 insertions(+)
 create mode 100755 tests/database/run-test.sh

diff --git a/tests/database/run-test.sh b/tests/database/run-test.sh
new file mode 100755
index 00000000..30c91863
--- /dev/null
+++ b/tests/database/run-test.sh
@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+    # Dump environment variables for debugging
+    echo "Environment variables:"
+    env
+fi
+
+if [ "$1" == "from-path" ]; then
+    TEST_TARGET_SO="laconic-so"
+else
+    TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+fi
+
+stack="test-database"
+spec_file=${stack}-spec.yml
+deployment_dir=${stack}-deployment
+
+# Helper functions: TODO move into a separate file
+wait_for_pods_started () {
+    for i in {1..50}
+    do
+        local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps )
+
+        if [[ "$ps_output" == *"Running containers:"* ]]; then
+            # if ready, return
+            return
+        else
+            # if not ready, wait
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "waiting for pods to start: FAILED"
+    delete_cluster_exit
+}
+
+wait_for_log_output () {
+    for i in {1..50}
+    do
+
+        local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+
+        if [[ ! -z "$log_output" ]]; then
+            # if ready, return
+            return
+        else
+            # if not ready, wait
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "waiting for pods log content: FAILED"
+    delete_cluster_exit
+}
+
+
+delete_cluster_exit () {
+    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+    exit 1
+}
+
+# Set a non-default repo dir
+export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
+echo "Testing this package: $TEST_TARGET_SO"
+echo "Test version command"
+reported_version_string=$( $TEST_TARGET_SO version )
+echo "Version reported is: ${reported_version_string}"
+echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
+rm -rf $CERC_REPO_BASE_DIR
+mkdir -p $CERC_REPO_BASE_DIR
+$TEST_TARGET_SO --stack ${stack} setup-repositories
+$TEST_TARGET_SO --stack ${stack} build-containers
+# Test basic stack-orchestrator deploy to k8s
+test_deployment_dir=$CERC_REPO_BASE_DIR/test-${deployment_dir}
+test_deployment_spec=$CERC_REPO_BASE_DIR/test-${spec_file}
+
+$TEST_TARGET_SO --stack ${stack} deploy --deploy-to k8s-kind init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED
+# Check the file now exists
+if [ ! -f "$test_deployment_spec" ]; then
+    echo "deploy init test: spec file not present"
+    echo "deploy init test: FAILED"
+    exit 1
+fi
+
+echo "deploy init test: passed"
+$TEST_TARGET_SO --stack ${stack} deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
+# Check the deployment dir exists
+if [ ! -d "$test_deployment_dir" ]; then
+    echo "deploy create test: deployment directory not present"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+echo "deploy create test: passed"
+
+# Try to start the deployment
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+wait_for_pods_started
+# Check logs command works
+wait_for_log_output
+log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then
+    echo "deployment logs test: passed"
+else
+    echo "deployment logs test: FAILED"
+    delete_cluster_exit
+fi
+
+# Stop and clean up
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+echo "Test passed"
-- 
2.45.2


From 7b5f720593ead246c0d44432147a142d8ee09d2b Mon Sep 17 00:00:00 2001
From: David Boreham <david@bozemanpass.com>
Date: Wed, 14 Feb 2024 15:29:33 -0700
Subject: [PATCH 3/9] Basic database test

---
 .../compose/docker-compose-test-database.yml  |  4 ++
 .../cerc-test-database-client/Dockerfile      | 12 ++++++
 .../cerc-test-database-client/build.sh        |  5 +++
 .../cerc-test-database-client/run.sh          | 42 +++++++++++++++++++
 .../data/stacks/test-database/stack.yml       |  1 +
 5 files changed, 64 insertions(+)
 create mode 100644 stack_orchestrator/data/container-build/cerc-test-database-client/Dockerfile
 create mode 100755 stack_orchestrator/data/container-build/cerc-test-database-client/build.sh
 create mode 100755 stack_orchestrator/data/container-build/cerc-test-database-client/run.sh

diff --git a/stack_orchestrator/data/compose/docker-compose-test-database.yml b/stack_orchestrator/data/compose/docker-compose-test-database.yml
index 7de4712b..6b99cdab 100644
--- a/stack_orchestrator/data/compose/docker-compose-test-database.yml
+++ b/stack_orchestrator/data/compose/docker-compose-test-database.yml
@@ -1,4 +1,5 @@
 services:
+
   database:
     image: cerc/test-database-container:local
     restart: always
@@ -11,6 +12,9 @@ services:
       POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C"
     ports:
       - "5432"
+    
+  test-client:
+    image: cerc/test-database-client:local
 
 volumes:
   db-data:
diff --git a/stack_orchestrator/data/container-build/cerc-test-database-client/Dockerfile b/stack_orchestrator/data/container-build/cerc-test-database-client/Dockerfile
new file mode 100644
index 00000000..e2b3f7ad
--- /dev/null
+++ b/stack_orchestrator/data/container-build/cerc-test-database-client/Dockerfile
@@ -0,0 +1,12 @@
+FROM ubuntu:latest
+
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive && export DEBCONF_NOWARNINGS="yes" && \
+    apt-get install -y software-properties-common && \
+    apt-get install -y postgresql-client && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+EXPOSE 80
+
+COPY run.sh /app/run.sh
+
+ENTRYPOINT ["/app/run.sh"]
diff --git a/stack_orchestrator/data/container-build/cerc-test-database-client/build.sh b/stack_orchestrator/data/container-build/cerc-test-database-client/build.sh
new file mode 100755
index 00000000..f9a9051f
--- /dev/null
+++ b/stack_orchestrator/data/container-build/cerc-test-database-client/build.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Build cerc/test-container
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+docker build -t cerc/test-database-client:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR
\ No newline at end of file
diff --git a/stack_orchestrator/data/container-build/cerc-test-database-client/run.sh b/stack_orchestrator/data/container-build/cerc-test-database-client/run.sh
new file mode 100755
index 00000000..611a8968
--- /dev/null
+++ b/stack_orchestrator/data/container-build/cerc-test-database-client/run.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# TODO derive this from config
+database_url="postgresql://test-user:password@localhost:5432/test-db"
+psql_command="psql ${database_url}"
+
+does_test_data_exist () {
+    query_result=$(${psql_command} -t -c "select count(*) from test_table_1 where key_column = 'test_key_1';" | head -1 | tr -d ' ')
+    if [[ "${query_result}" == "1" ]]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+create_test_data () {
+    ${psql_command} -c "create table test_table_1 (key_column text, value_column text, primary key(key_column));"
+    ${psql_command} -c "insert into test_table_1 values ('test_key_1', 'test_value_1');"
+}
+
+wait_forever() {
+    # Loop to keep docker/k8s happy since this is the container entrypoint
+    while :; do sleep 600; done
+}
+
+# Check if the test database content exists already
+if does_test_data_exist; then
+    # If so, log saying so. Test harness will look for this log output
+    echo "Database test client: test data already exists"
+else
+    # Otherwise log saying the content was not present
+    echo "Database test client: test data does not exist"
+    echo "Database test client: creating test data"
+    # then create it
+    create_test_data
+fi
+
+wait_forever
diff --git a/stack_orchestrator/data/stacks/test-database/stack.yml b/stack_orchestrator/data/stacks/test-database/stack.yml
index 6fac54e0..46fef720 100644
--- a/stack_orchestrator/data/stacks/test-database/stack.yml
+++ b/stack_orchestrator/data/stacks/test-database/stack.yml
@@ -4,5 +4,6 @@ description: "A test database stack"
 repos:
 containers:
   - cerc/test-database-container
+  - cerc/test-database-client
 pods:
   - test-database
-- 
2.45.2


From afadc83966daa9128b0ae83077860b6f84ecf3c8 Mon Sep 17 00:00:00 2001
From: David Boreham <david@bozemanpass.com>
Date: Wed, 14 Feb 2024 15:58:25 -0700
Subject: [PATCH 4/9] Add function to enumerate containers in a pod

---
 stack_orchestrator/deploy/k8s/helpers.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py
index 1fe697e7..d10f73b9 100644
--- a/stack_orchestrator/deploy/k8s/helpers.py
+++ b/stack_orchestrator/deploy/k8s/helpers.py
@@ -62,6 +62,17 @@ def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
     return pods
 
 
+def containers_in_pod(core_api: client.CoreV1Api, pod_name: str):
+    containers = []
+    pod_response = core_api.read_namespaced_pod(pod_name, namespace="default")
+    if opts.o.debug:
+        print(f"pod_response: {pod_response}")
+    pod_containers = pod_response.spec.containers
+    for pod_container in pod_containers:
+        containers.append(pod_container.name)
+    return containers
+
+
 def log_stream_from_string(s: str):
     # Note response has to be UTF-8 encoded because the caller expects to decode it
     yield ("ignore", s.encode())
-- 
2.45.2


From 0fbdb18a76462ad8563d7c767fd92f18ac5508cb Mon Sep 17 00:00:00 2001
From: David Boreham <david@bozemanpass.com>
Date: Wed, 14 Feb 2024 15:59:03 -0700
Subject: [PATCH 5/9] Fix logging for multi-container pods

---
 stack_orchestrator/deploy/k8s/deploy_k8s.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py
index e1f729c5..b4a6a853 100644
--- a/stack_orchestrator/deploy/k8s/deploy_k8s.py
+++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py
@@ -20,7 +20,7 @@ from kubernetes import client, config
 from stack_orchestrator import constants
 from stack_orchestrator.deploy.deployer import Deployer, DeployerConfigGenerator
 from stack_orchestrator.deploy.k8s.helpers import create_cluster, destroy_cluster, load_images_into_kind
-from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, log_stream_from_string, generate_kind_config
+from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, containers_in_pod, log_stream_from_string, generate_kind_config
 from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo
 from stack_orchestrator.opts import opts
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
@@ -359,9 +359,15 @@ class K8sDeployer(Deployer):
             log_data = "******* Pods not running ********\n"
         else:
             k8s_pod_name = pods[0]
+            containers = containers_in_pod(self.core_api, k8s_pod_name)
             # If the pod is not yet started, the logs request below will throw an exception
             try:
-                log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
+                log_data = ""
+                for container in containers:
+                    container_log = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container=container)
+                    container_log_lines = container_log.splitlines()
+                    for line in container_log_lines:
+                        log_data += f"{container}: {line}\n"
             except client.exceptions.ApiException as e:
                 if opts.o.debug:
                     print(f"Error from read_namespaced_pod_log: {e}")
-- 
2.45.2


From d49f366b5650c38ab1e64b9a5d43bbeba073ebf9 Mon Sep 17 00:00:00 2001
From: David Boreham <david@bozemanpass.com>
Date: Wed, 14 Feb 2024 16:12:10 -0700
Subject: [PATCH 6/9] Flesh out test

---
 tests/database/run-test.sh | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/tests/database/run-test.sh b/tests/database/run-test.sh
index 30c91863..8fe4be2e 100755
--- a/tests/database/run-test.sh
+++ b/tests/database/run-test.sh
@@ -76,15 +76,15 @@ $TEST_TARGET_SO --stack ${stack} build-containers
 test_deployment_dir=$CERC_REPO_BASE_DIR/test-${deployment_dir}
 test_deployment_spec=$CERC_REPO_BASE_DIR/test-${spec_file}
 
-$TEST_TARGET_SO --stack ${stack} deploy --deploy-to k8s-kind init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED
+$TEST_TARGET_SO --stack ${stack} deploy --deploy-to k8s-kind init --output $test_deployment_spec
 # Check the file now exists
 if [ ! -f "$test_deployment_spec" ]; then
     echo "deploy init test: spec file not present"
     echo "deploy init test: FAILED"
     exit 1
 fi
-
 echo "deploy init test: passed"
+
 $TEST_TARGET_SO --stack ${stack} deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
 # Check the deployment dir exists
 if [ ! -d "$test_deployment_dir" ]; then
@@ -99,14 +99,30 @@ $TEST_TARGET_SO deployment --dir $test_deployment_dir start
 wait_for_pods_started
 # Check logs command works
 wait_for_log_output
-log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then
+log_output_1=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_1" == *"Database test client: test data does not exist"* ]]; then
     echo "deployment logs test: passed"
 else
     echo "deployment logs test: FAILED"
     delete_cluster_exit
 fi
 
+# Stop then start again and check the volume was preserved
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
+# Sleep a bit just in case
+sleep 20
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+wait_for_pods_started
+wait_for_log_output
+
+log_output_2=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_2" == *"Database test client: test data already exists"* ]]; then
+    echo "Retain database content test: passed"
+else
+    echo "Retain database content test: FAILED"
+    delete_cluster_exit
+fi
+
 # Stop and clean up
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
 echo "Test passed"
-- 
2.45.2


From 43be190fe629fde805dbd20665e9940e5f207b41 Mon Sep 17 00:00:00 2001
From: David Boreham <david@bozemanpass.com>
Date: Wed, 14 Feb 2024 21:49:16 -0700
Subject: [PATCH 7/9] Test complete

---
 .../cerc-test-database-client/run.sh          | 35 +++++++++++++++++--
 tests/database/run-test.sh                    | 14 ++++----
 2 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/stack_orchestrator/data/container-build/cerc-test-database-client/run.sh b/stack_orchestrator/data/container-build/cerc-test-database-client/run.sh
index 611a8968..b889d677 100755
--- a/stack_orchestrator/data/container-build/cerc-test-database-client/run.sh
+++ b/stack_orchestrator/data/container-build/cerc-test-database-client/run.sh
@@ -7,6 +7,32 @@ fi
 # TODO derive this from config
 database_url="postgresql://test-user:password@localhost:5432/test-db"
 psql_command="psql ${database_url}"
+program_name="Database test client:"
+
+wait_for_database_up () {
+    for i in {1..50}
+    do
+        ${psql_command} -c "select 1;"
+        psql_succeeded=$?
+        if [[ ${psql_succeeded} == 0 ]]; then
+            # if ready, return
+            echo "${program_name} database up"
+            return
+        else
+            # if not ready, wait
+            echo "${program_name} waiting for database: ${i}"
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "${program_name} waiting for database: FAILED"
+    exit 1
+}
+
+# Used to synchronize with the test runner
+notify_test_complete () {
+    echo "${program_name} test complete"
+}
 
 does_test_data_exist () {
     query_result=$(${psql_command} -t -c "select count(*) from test_table_1 where key_column = 'test_key_1';" | head -1 | tr -d ' ')
@@ -27,16 +53,19 @@ wait_forever() {
     while :; do sleep 600; done
 }
 
+wait_for_database_up
+
 # Check if the test database content exists already
 if does_test_data_exist; then
     # If so, log saying so. Test harness will look for this log output
-    echo "Database test client: test data already exists"
+    echo "${program_name} test data already exists"
 else
     # Otherwise log saying the content was not present
-    echo "Database test client: test data does not exist"
-    echo "Database test client: creating test data"
+    echo "${program_name} test data does not exist"
+    echo "${program_name} creating test data"
     # then create it
     create_test_data
 fi
 
+notify_test_complete
 wait_forever
diff --git a/tests/database/run-test.sh b/tests/database/run-test.sh
index 8fe4be2e..e6aca59c 100755
--- a/tests/database/run-test.sh
+++ b/tests/database/run-test.sh
@@ -36,13 +36,13 @@ wait_for_pods_started () {
     delete_cluster_exit
 }
 
-wait_for_log_output () {
+wait_for_test_complete () {
     for i in {1..50}
     do
 
         local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
 
-        if [[ ! -z "$log_output" ]]; then
+    if [[ "${log_output}" == *"Database test client: test complete"* ]]; then
             # if ready, return
             return
         else
@@ -51,7 +51,7 @@ wait_for_log_output () {
         fi
     done
     # Timed out, error exit
-    echo "waiting for pods log content: FAILED"
+    echo "waiting for test complete: FAILED"
     delete_cluster_exit
 }
 
@@ -98,12 +98,12 @@ echo "deploy create test: passed"
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
 wait_for_pods_started
 # Check logs command works
-wait_for_log_output
+wait_for_test_complete
 log_output_1=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
 if [[ "$log_output_1" == *"Database test client: test data does not exist"* ]]; then
-    echo "deployment logs test: passed"
+    echo "Create database content test: passed"
 else
-    echo "deployment logs test: FAILED"
+    echo "Create database content test: FAILED"
     delete_cluster_exit
 fi
 
@@ -113,7 +113,7 @@ $TEST_TARGET_SO deployment --dir $test_deployment_dir stop
 sleep 20
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
 wait_for_pods_started
-wait_for_log_output
+wait_for_test_complete
 
 log_output_2=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
 if [[ "$log_output_2" == *"Database test client: test data already exists"* ]]; then
-- 
2.45.2


From 511fa87ec42112d99b75ddf546c8e36d97c25ca9 Mon Sep 17 00:00:00 2001
From: David Boreham <david@bozemanpass.com>
Date: Wed, 14 Feb 2024 21:58:23 -0700
Subject: [PATCH 8/9] Fix lint error

---
 stack_orchestrator/deploy/k8s/deploy_k8s.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py
index 0976b1aa..db806050 100644
--- a/stack_orchestrator/deploy/k8s/deploy_k8s.py
+++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py
@@ -20,7 +20,8 @@ from kubernetes import client, config
 from stack_orchestrator import constants
 from stack_orchestrator.deploy.deployer import Deployer, DeployerConfigGenerator
 from stack_orchestrator.deploy.k8s.helpers import create_cluster, destroy_cluster, load_images_into_kind
-from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, containers_in_pod, log_stream_from_string, generate_kind_config
+from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, containers_in_pod, log_stream_from_string
+from stack_orchestrator.deploy.k8s.helpers import generate_kind_config
 from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo
 from stack_orchestrator.opts import opts
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
-- 
2.45.2


From cdf0559b6732f88522a17d0a260a4cf6a0970aed Mon Sep 17 00:00:00 2001
From: David Boreham <david@bozemanpass.com>
Date: Wed, 14 Feb 2024 22:07:09 -0700
Subject: [PATCH 9/9] Add CI job

---
 .gitea/workflows/test-database-yml      | 52 +++++++++++++++++++++++++
 .gitea/workflows/triggers/test-database |  1 +
 2 files changed, 53 insertions(+)
 create mode 100644 .gitea/workflows/test-database-yml
 create mode 100644 .gitea/workflows/triggers/test-database

diff --git a/.gitea/workflows/test-database-yml b/.gitea/workflows/test-database-yml
new file mode 100644
index 00000000..b925271b
--- /dev/null
+++ b/.gitea/workflows/test-database-yml
@@ -0,0 +1,52 @@
+name: Database Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-database'
+      - '.gitea/workflows/test-database.yml'
+      - 'tests/database/run-test.sh'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '5 18 * * *'
+
+jobs:
+  test:
+    name: "Run database hosting test on kind/k8s"
+    runs-on: ubuntu-22.04
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Run database deployment test"
+        run: |
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/database/run-test.sh
+
diff --git a/.gitea/workflows/triggers/test-database b/.gitea/workflows/triggers/test-database
new file mode 100644
index 00000000..7b6fbcaf
--- /dev/null
+++ b/.gitea/workflows/triggers/test-database
@@ -0,0 +1 @@
+Change this file to trigger running the test-database CI job
-- 
2.45.2