Merge branch 'main' into ci-test

Use absolute path for the data volume (#749 )
Reviewed-on: cerc-io/stack-orchestrator#749 Co-authored-by: David Boreham <david@bozemanpass.com> Co-committed-by: David Boreham <david@bozemanpass.com>
2024-02-17 07:30:29 -07:00 · 2024-02-17 14:29:53 +00:00 · 2024-02-17 07:24:12 -07:00 · 2024-02-17 01:54:30 +00:00 · 2024-02-16 04:11:09 +00:00 · 2024-02-15 08:41:54 -07:00
34 changed files with 1501 additions and 156 deletions
@@ -0,0 +1,52 @@
+name: Database Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-database'
+      - '.gitea/workflows/test-database.yml'
+      - 'tests/database/run-test.sh'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '5 18 * * *'
+
+jobs:
+  test:
+    name: "Run database hosting test on kind/k8s"
+    runs-on: ubuntu-22.04
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Run database deployment test"
+        run: |
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          CERC_SCRIPT_DEBUG=1 ./tests/database/run-test.sh
+
@@ -1,6 +1,8 @@
 name: K8s Deploy Test

 on:
+  pull_request:
+    branches: '*'
  push:
    branches: '*'
    paths:
@@ -0,0 +1,2 @@
+Change this file to trigger running the test-database CI job
+Trigger test run
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Beginnings of a script to quickly spin up and test a deployment
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+if [[ -n "$1" ]]; then
+	stack_name=$1
+else
+	stack_name="test"
+fi
+spec_file_name="${stack_name}-spec.yml"
+deployment_dir_name="${stack_name}-deployment"
+rm -f ${spec_file_name}
+rm -rf ${deployment_dir_name}
+laconic-so --stack ${stack_name} deploy --deploy-to k8s-kind init --output ${spec_file_name}
+laconic-so --stack ${stack_name} deploy --deploy-to k8s-kind create --deployment-dir ${deployment_dir_name} --spec-file ${spec_file_name}
+#laconic-so deployment --dir ${deployment_dir_name} start
+#laconic-so deployment --dir ${deployment_dir_name} ps
+#laconic-so deployment --dir ${deployment_dir_name} stop
@@ -27,6 +27,12 @@ kube_config_key = "kube-config"
 deploy_to_key = "deploy-to"
 network_key = "network"
 http_proxy_key = "http-proxy"
-image_resigtry_key = "image-registry"
+image_registry_key = "image-registry"
+configmaps_key = "configmaps"
+resources_key = "resources"
+volumes_key = "volumes"
+security_key = "security"
+annotations_key = "annotations"
+labels_key = "labels"
 kind_config_filename = "kind-config.yml"
 kube_config_filename = "kubeconfig.yml"
@@ -0,0 +1,20 @@
+services:
+
+  database:
+    image: cerc/test-database-container:local
+    restart: always
+    volumes:
+      - db-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: "test-user"
+      POSTGRES_DB: "test-db"
+      POSTGRES_PASSWORD: "password"
+      POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C"
+    ports:
+      - "5432"
+    
+  test-client:
+    image: cerc/test-database-client:local
+
+volumes:
+  db-data:
@@ -7,11 +7,13 @@ services:
      CERC_TEST_PARAM_1: ${CERC_TEST_PARAM_1:-FAILED}
      CERC_TEST_PARAM_2: "CERC_TEST_PARAM_2_VALUE"
    volumes:
-      - test-data:/data
+      - test-data-bind:/data
+      - test-data-auto:/data2
      - test-config:/config:ro
    ports:
      - "80"

 volumes:
-  test-data:
+  test-data-bind:
+  test-data-auto:
  test-config:
@@ -5,4 +5,4 @@ services:
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
    ports:
-      - "3000"
+      - "80"
@@ -33,8 +33,8 @@ if [ -f ".env" ]; then
  rm -f $TMP_ENV
 fi

-for f in $(find "$TRG_DIR" -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do
-  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},();]' '\n' | egrep -o '^"CERC_RUNTIME_ENV_[^\"]+"'); do
+for f in $(find . -type f \( -regex '.*.html?' -or -regex ".*.[tj]s\(x\|on\)?$" \) | grep -v 'node_modules' | grep -v '.git'); do
+  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '["/\\{},();]' '\n' | tr -s "[']" '\n' | egrep -o -e '^CERC_RUNTIME_ENV_.+$' -e '^LACONIC_HOSTED_CONFIG_.+$'); do
    orig_name=$(echo -n "${e}" | sed 's/"//g')
    cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
    cur_val=$(echo -n "\$${cur_name}" | envsubst)
@@ -1,19 +1,38 @@
 #!/usr/bin/env bash
 set -e
+
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
  set -x
 fi
-# Test if the container's filesystem is old (run previously) or new
-EXISTSFILENAME=/data/exists
+
 echo "Test container starting"
-if [[ -f "$EXISTSFILENAME" ]];
-then
-    TIMESTAMP=`cat $EXISTSFILENAME`
-    echo "Filesystem is old, created: $TIMESTAMP" 
+
+DATA_DEVICE=$(df | grep "/data$" | awk '{ print $1 }')
+if [[ -n "$DATA_DEVICE" ]]; then
+  echo "/data: MOUNTED dev=${DATA_DEVICE}"
 else
-    echo "Filesystem is fresh"
-    echo `date` > $EXISTSFILENAME
+  echo "/data: not mounted"
 fi
+
+DATA2_DEVICE=$(df | grep "/data2$" | awk '{ print $1 }')
+if [[ -n "$DATA_DEVICE" ]]; then
+  echo "/data2: MOUNTED dev=${DATA2_DEVICE}"
+else
+  echo "/data2: not mounted"
+fi
+
+# Test if the container's filesystem is old (run previously) or new
+for d in /data /data2; do
+  if [[ -f "$d/exists" ]];
+  then
+      TIMESTAMP=`cat $d/exists`
+      echo "$d filesystem is old, created: $TIMESTAMP"
+  else
+      echo "$d filesystem is fresh"
+      echo `date` > $d/exists
+  fi
+done
+
 if [ -n "$CERC_TEST_PARAM_1" ]; then
  echo "Test-param-1: ${CERC_TEST_PARAM_1}"
 fi
@@ -0,0 +1,12 @@
+FROM ubuntu:latest
+
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive && export DEBCONF_NOWARNINGS="yes" && \
+    apt-get install -y software-properties-common && \
+    apt-get install -y postgresql-client && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+EXPOSE 80
+
+COPY run.sh /app/run.sh
+
+ENTRYPOINT ["/app/run.sh"]
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Build cerc/test-container
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+docker build -t cerc/test-database-client:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# TODO derive this from config
+database_url="postgresql://test-user:password@localhost:5432/test-db"
+psql_command="psql ${database_url}"
+program_name="Database test client:"
+
+wait_for_database_up () {
+    for i in {1..50}
+    do
+        ${psql_command} -c "select 1;"
+        psql_succeeded=$?
+        if [[ ${psql_succeeded} == 0 ]]; then
+            # if ready, return
+            echo "${program_name} database up"
+            return
+        else
+            # if not ready, wait
+            echo "${program_name} waiting for database: ${i}"
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "${program_name} waiting for database: FAILED"
+    exit 1
+}
+
+# Used to synchronize with the test runner
+notify_test_complete () {
+    echo "${program_name} test complete"
+}
+
+does_test_data_exist () {
+    query_result=$(${psql_command} -t -c "select count(*) from test_table_1 where key_column = 'test_key_1';" | head -1 | tr -d ' ')
+    if [[ "${query_result}" == "1" ]]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+create_test_data () {
+    ${psql_command} -c "create table test_table_1 (key_column text, value_column text, primary key(key_column));"
+    ${psql_command} -c "insert into test_table_1 values ('test_key_1', 'test_value_1');"
+}
+
+wait_forever() {
+    # Loop to keep docker/k8s happy since this is the container entrypoint
+    while :; do sleep 600; done
+}
+
+wait_for_database_up
+
+# Check if the test database content exists already
+if does_test_data_exist; then
+    # If so, log saying so. Test harness will look for this log output
+    echo "${program_name} test data already exists"
+else
+    # Otherwise log saying the content was not present
+    echo "${program_name} test data does not exist"
+    echo "${program_name} creating test data"
+    # then create it
+    create_test_data
+fi
+
+notify_test_complete
+wait_forever
@@ -0,0 +1,3 @@
+FROM postgres:16-bullseye
+
+EXPOSE 5432
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Build cerc/test-container
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+docker build -t cerc/test-database-container:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR
@@ -18,8 +18,8 @@ if [ -f ".env" ]; then
  rm -f $TMP_ENV
 fi

-for f in $(find . -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do
-  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},();]' '\n' | egrep -o -e '^"CERC_RUNTIME_ENV_[^\"]+"' -e '^"LACONIC_HOSTED_CONFIG_[^\"]+"'); do
+for f in $(find . -type f \( -regex '.*.html?' -or -regex ".*.[tj]s\(x\|on\)?$" \) | grep -v 'node_modules' | grep -v '.git'); do
+  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '["/\\{},();]' '\n' | tr -s "[']" '\n' | egrep -o -e '^CERC_RUNTIME_ENV_.+$' -e '^LACONIC_HOSTED_CONFIG_.+$'); do
    orig_name=$(echo -n "${e}" | sed 's/"//g')
    cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
    cur_val=$(echo -n "\$${cur_name}" | envsubst)
@@ -33,4 +33,10 @@ else
  mv "${WORK_DIR}" "${DEST_DIR}"
 fi

+# One special fix ...
+cd "${DEST_DIR}"
+for f in $(find . -type f -name '*.htm*'); do
+  sed -i -e 's#/LACONIC_HOSTED_CONFIG_homepage/#LACONIC_HOSTED_CONFIG_homepage/#g' "$f"
+done
+
 exit 0
@@ -0,0 +1,673 @@
+# from: https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
+# via: https://kind.sigs.k8s.io/docs/user/ingress/#ingress-nginx
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+  name: ingress-nginx
+---
+apiVersion: v1
+automountServiceAccountToken: true
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx
+  namespace: ingress-nginx
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx
+  namespace: ingress-nginx
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - pods
+  - secrets
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses/status
+  verbs:
+  - update
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingressclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - coordination.k8s.io
+  resourceNames:
+  - ingress-nginx-leader
+  resources:
+  - leases
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - list
+  - watch
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - endpoints
+  - nodes
+  - pods
+  - secrets
+  - namespaces
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses/status
+  verbs:
+  - update
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingressclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - list
+  - watch
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+rules:
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - validatingwebhookconfigurations
+  verbs:
+  - get
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx
+  namespace: ingress-nginx
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ingress-nginx
+subjects:
+- kind: ServiceAccount
+  name: ingress-nginx
+  namespace: ingress-nginx
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ingress-nginx-admission
+subjects:
+- kind: ServiceAccount
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ingress-nginx
+subjects:
+- kind: ServiceAccount
+  name: ingress-nginx
+  namespace: ingress-nginx
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ingress-nginx-admission
+subjects:
+- kind: ServiceAccount
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+---
+apiVersion: v1
+data:
+  allow-snippet-annotations: "false"
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+spec:
+  ipFamilies:
+  - IPv4
+  ipFamilyPolicy: SingleStack
+  ports:
+  - appProtocol: http
+    name: http
+    port: 80
+    protocol: TCP
+    targetPort: http
+  - appProtocol: https
+    name: https
+    port: 443
+    protocol: TCP
+    targetPort: https
+  selector:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+  type: NodePort
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-controller-admission
+  namespace: ingress-nginx
+spec:
+  ports:
+  - appProtocol: https
+    name: https-webhook
+    port: 443
+    targetPort: webhook
+  selector:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+spec:
+  minReadySeconds: 0
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: controller
+      app.kubernetes.io/instance: ingress-nginx
+      app.kubernetes.io/name: ingress-nginx
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: controller
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/part-of: ingress-nginx
+        app.kubernetes.io/version: 1.9.6
+    spec:
+      containers:
+      - args:
+        - /nginx-ingress-controller
+        - --election-id=ingress-nginx-leader
+        - --controller-class=k8s.io/ingress-nginx
+        - --ingress-class=nginx
+        - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
+        - --validating-webhook=:8443
+        - --validating-webhook-certificate=/usr/local/certificates/cert
+        - --validating-webhook-key=/usr/local/certificates/key
+        - --watch-ingress-without-class=true
+        - --publish-status-address=localhost
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: LD_PRELOAD
+          value: /usr/local/lib/libmimalloc.so
+        image: registry.k8s.io/ingress-nginx/controller:v1.9.6@sha256:1405cc613bd95b2c6edd8b2a152510ae91c7e62aea4698500d23b2145960ab9c
+        imagePullPolicy: IfNotPresent
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - /wait-shutdown
+        livenessProbe:
+          failureThreshold: 5
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+        name: controller
+        ports:
+        - containerPort: 80
+          hostPort: 80
+          name: http
+          protocol: TCP
+        - containerPort: 443
+          hostPort: 443
+          name: https
+          protocol: TCP
+        - containerPort: 8443
+          name: webhook
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+        resources:
+          requests:
+            cpu: 100m
+            memory: 90Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - ALL
+          readOnlyRootFilesystem: false
+          runAsNonRoot: true
+          runAsUser: 101
+          seccompProfile:
+            type: RuntimeDefault
+        volumeMounts:
+        - mountPath: /usr/local/certificates/
+          name: webhook-cert
+          readOnly: true
+      dnsPolicy: ClusterFirst
+      nodeSelector:
+        ingress-ready: "true"
+        kubernetes.io/os: linux
+      serviceAccountName: ingress-nginx
+      terminationGracePeriodSeconds: 0
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+        operator: Equal
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+        operator: Equal
+      volumes:
+      - name: webhook-cert
+        secret:
+          secretName: ingress-nginx-admission
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission-create
+  namespace: ingress-nginx
+spec:
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: admission-webhook
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/part-of: ingress-nginx
+        app.kubernetes.io/version: 1.9.6
+      name: ingress-nginx-admission-create
+    spec:
+      containers:
+      - args:
+        - create
+        - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc
+        - --namespace=$(POD_NAMESPACE)
+        - --secret-name=ingress-nginx-admission
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20231226-1a7112e06@sha256:25d6a5f11211cc5c3f9f2bf552b585374af287b4debf693cacbe2da47daa5084
+        imagePullPolicy: IfNotPresent
+        name: create
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 65532
+          seccompProfile:
+            type: RuntimeDefault
+      nodeSelector:
+        kubernetes.io/os: linux
+      restartPolicy: OnFailure
+      serviceAccountName: ingress-nginx-admission
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission-patch
+  namespace: ingress-nginx
+spec:
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: admission-webhook
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/part-of: ingress-nginx
+        app.kubernetes.io/version: 1.9.6
+      name: ingress-nginx-admission-patch
+    spec:
+      containers:
+      - args:
+        - patch
+        - --webhook-name=ingress-nginx-admission
+        - --namespace=$(POD_NAMESPACE)
+        - --patch-mutating=false
+        - --secret-name=ingress-nginx-admission
+        - --patch-failure-policy=Fail
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20231226-1a7112e06@sha256:25d6a5f11211cc5c3f9f2bf552b585374af287b4debf693cacbe2da47daa5084
+        imagePullPolicy: IfNotPresent
+        name: patch
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 65532
+          seccompProfile:
+            type: RuntimeDefault
+      nodeSelector:
+        kubernetes.io/os: linux
+      restartPolicy: OnFailure
+      serviceAccountName: ingress-nginx-admission
+---
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: nginx
+spec:
+  controller: k8s.io/ingress-nginx
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: ingress-nginx-controller-admission
+      namespace: ingress-nginx
+      path: /networking/v1/ingresses
+  failurePolicy: Fail
+  matchPolicy: Equivalent
+  name: validate.nginx.ingress.kubernetes.io
+  rules:
+  - apiGroups:
+    - networking.k8s.io
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - ingresses
+  sideEffects: None
@@ -0,0 +1,3 @@
+# Test Database Stack
+
+A stack with a database for test/demo purposes.
@@ -0,0 +1,9 @@
+version: "1.0"
+name: test
+description: "A test database stack"
+repos:
+containers:
+  - cerc/test-database-container
+  - cerc/test-database-client
+pods:
+  - test-database
@@ -27,6 +27,7 @@ from stack_orchestrator.opts import opts
 from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
                                     global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
                                     get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file)
+from stack_orchestrator.deploy.spec import Spec
 from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand
 from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
@@ -111,6 +112,7 @@ def _create_bind_dir_if_relative(volume, path_string, compose_dir):

 # See: https://stackoverflow.com/questions/45699189/editing-docker-compose-yml-with-pyyaml
 def _fixup_pod_file(pod, spec, compose_dir):
+    deployment_type = spec[constants.deploy_to_key]
    # Fix up volumes
    if "volumes" in spec:
        spec_volumes = spec["volumes"]
@@ -119,27 +121,34 @@ def _fixup_pod_file(pod, spec, compose_dir):
            for volume in pod_volumes.keys():
                if volume in spec_volumes:
                    volume_spec = spec_volumes[volume]
-                    volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}"
-                    _create_bind_dir_if_relative(volume, volume_spec, compose_dir)
-                    new_volume_spec = {"driver": "local",
-                                       "driver_opts": {
-                                           "type": "none",
-                                           "device": volume_spec_fixedup,
-                                           "o": "bind"
-                                       }
-                                       }
-                    pod["volumes"][volume] = new_volume_spec
+                    if volume_spec:
+                        volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}"
+                        _create_bind_dir_if_relative(volume, volume_spec, compose_dir)
+                        # this is Docker specific
+                        if spec.is_docker_deployment():
+                            new_volume_spec = {
+                                "driver": "local",
+                                "driver_opts": {
+                                    "type": "none",
+                                    "device": volume_spec_fixedup,
+                                    "o": "bind"
+                                }
+                            }
+                            pod["volumes"][volume] = new_volume_spec

    # Fix up configmaps
-    if "configmaps" in spec:
-        spec_cfgmaps = spec["configmaps"]
-        if "volumes" in pod:
-            pod_volumes = pod["volumes"]
-            for volume in pod_volumes.keys():
-                if volume in spec_cfgmaps:
-                    volume_cfg = spec_cfgmaps[volume]
-                    # Just make the dir (if necessary)
-                    _create_bind_dir_if_relative(volume, volume_cfg, compose_dir)
+    if constants.configmaps_key in spec:
+        if spec.is_kubernetes_deployment():
+            spec_cfgmaps = spec[constants.configmaps_key]
+            if "volumes" in pod:
+                pod_volumes = pod[constants.volumes_key]
+                for volume in pod_volumes.keys():
+                    if volume in spec_cfgmaps:
+                        volume_cfg = spec_cfgmaps[volume]
+                        # Just make the dir (if necessary)
+                        _create_bind_dir_if_relative(volume, volume_cfg, compose_dir)
+        else:
+            print(f"Warning: ConfigMaps not supported for {deployment_type}")

    # Fix up ports
    if "network" in spec and "ports" in spec["network"]:
@@ -323,7 +332,7 @@ def init_operation(deploy_command_context, stack, deployer_type, config,
        if image_registry is None:
            error_exit("--image-registry must be supplied with --deploy-to k8s")
        spec_file_content.update({constants.kube_config_key: kube_config})
-        spec_file_content.update({constants.image_resigtry_key: image_registry})
+        spec_file_content.update({constants.image_registry_key: image_registry})
    else:
        # Check for --kube-config supplied for non-relevant deployer types
        if kube_config is not None:
@@ -358,10 +367,16 @@ def init_operation(deploy_command_context, stack, deployer_type, config,
        volume_descriptors = {}
        configmap_descriptors = {}
        for named_volume in named_volumes["rw"]:
-            volume_descriptors[named_volume] = f"./data/{named_volume}"
+            if "k8s" in deployer_type:
+                volume_descriptors[named_volume] = None
+            else:
+                volume_descriptors[named_volume] = f"./data/{named_volume}"
        for named_volume in named_volumes["ro"]:
-            if "k8s" in deployer_type and "config" in named_volume:
-                configmap_descriptors[named_volume] = f"./data/{named_volume}"
+            if "k8s" in deployer_type:
+                if "config" in named_volume:
+                    configmap_descriptors[named_volume] = f"./configmaps/{named_volume}"
+                else:
+                    volume_descriptors[named_volume] = None
            else:
                volume_descriptors[named_volume] = f"./data/{named_volume}"
        if volume_descriptors:
@@ -406,6 +421,17 @@ def _create_deployment_file(deployment_dir: Path):
        output_file.write(f"{constants.cluster_id_key}: {cluster}\n")


+def _check_volume_definitions(spec):
+    if spec.is_kubernetes_deployment():
+        for volume_name, volume_path in spec.get_volumes().items():
+            if volume_path:
+                if not os.path.isabs(volume_path):
+                    raise Exception(
+                        f"Relative path {volume_path} for volume {volume_name} not "
+                        f"supported for deployment type {spec.get_deployment_type()}"
+                    )
+
+
@click.command()
@click.option("--spec-file", required=True, help="Spec file to use to create this deployment")
@click.option("--deployment-dir", help="Create deployment files in this directory")
@@ -421,7 +447,8 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
 # The init command's implementation is in a separate function so that we can
 # call it from other commands, bypassing the click decoration stuff
 def create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers):
-    parsed_spec = get_parsed_deployment_spec(spec_file)
+    parsed_spec = Spec(os.path.abspath(spec_file), get_parsed_deployment_spec(spec_file))
+    _check_volume_definitions(parsed_spec)
    stack_name = parsed_spec["stack"]
    deployment_type = parsed_spec[constants.deploy_to_key]
    stack_file = get_stack_file_path(stack_name)
@@ -46,7 +46,7 @@ def push_images_operation(command_context: DeployCommandContext, deployment_cont
    cluster_context = command_context.cluster_context
    images: Set[str] = images_for_deployment(cluster_context.compose_files)
    # Tag the images for the remote repo
-    remote_repo_url = deployment_context.spec.obj[constants.image_resigtry_key]
+    remote_repo_url = deployment_context.spec.obj[constants.image_registry_key]
    docker = DockerClient()
    for image in images:
        if _image_needs_pushed(image):
@@ -21,7 +21,7 @@ from typing import Any, List, Set
 from stack_orchestrator.opts import opts
 from stack_orchestrator.util import env_var_map_from_file
 from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files
-from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path
+from stack_orchestrator.deploy.k8s.helpers import get_kind_pv_bind_mount_path
 from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map, envs_from_compose_file, merge_envs
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment
 from stack_orchestrator.deploy.deploy_types import DeployEnvVars
@@ -78,7 +78,30 @@ class ClusterInfo:
        if (opts.o.debug):
            print(f"Env vars: {self.environment_variables.map}")

-    def get_ingress(self):
+    def get_nodeport(self):
+        for pod_name in self.parsed_pod_yaml_map:
+            pod = self.parsed_pod_yaml_map[pod_name]
+            services = pod["services"]
+            for service_name in services:
+                service_info = services[service_name]
+                if "ports" in service_info:
+                    port = int(service_info["ports"][0])
+                    if opts.o.debug:
+                        print(f"service port: {port}")
+        service = client.V1Service(
+            metadata=client.V1ObjectMeta(name=f"{self.app_name}-nodeport"),
+            spec=client.V1ServiceSpec(
+                type="NodePort",
+                ports=[client.V1ServicePort(
+                    port=port,
+                    target_port=port
+                )],
+                selector={"app": self.app_name}
+            )
+        )
+        return service
+
+    def get_ingress(self, use_tls=False):
        # No ingress for a deployment that has no http-proxy defined, for now
        http_proxy_info_list = self.spec.get_http_proxy()
        ingress = None
@@ -93,7 +116,7 @@ class ClusterInfo:
            tls = [client.V1IngressTLS(
                hosts=[host_name],
                secret_name=f"{self.app_name}-tls"
-            )]
+            )] if use_tls else None
            paths = []
            for route in http_proxy_info["routes"]:
                path = route["path"]
@@ -171,21 +194,33 @@ class ClusterInfo:
            print(f"Spec Volumes: {spec_volumes}")
            print(f"Named Volumes: {named_volumes}")
            print(f"Resources: {resources}")
-        for volume_name in spec_volumes:
+        for volume_name, volume_path in spec_volumes.items():
            if volume_name not in named_volumes:
                if opts.o.debug:
                    print(f"{volume_name} not in pod files")
                continue
+
+            labels = {
+                "app": self.app_name,
+                "volume-label": f"{self.app_name}-{volume_name}"
+            }
+            if volume_path:
+                storage_class_name = "manual"
+                k8s_volume_name = f"{self.app_name}-{volume_name}"
+            else:
+                # These will be auto-assigned.
+                storage_class_name = None
+                k8s_volume_name = None
+
            spec = client.V1PersistentVolumeClaimSpec(
                access_modes=["ReadWriteOnce"],
-                storage_class_name="manual",
+                storage_class_name=storage_class_name,
                resources=to_k8s_resource_requirements(resources),
-                volume_name=f"{self.app_name}-{volume_name}"
+                volume_name=k8s_volume_name
            )
            pvc = client.V1PersistentVolumeClaim(
-                metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}",
-                                             labels={"volume-label": f"{self.app_name}-{volume_name}"}),
-                spec=spec,
+                metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}", labels=labels),
+                spec=spec
            )
            result.append(pvc)
        return result
@@ -226,16 +261,32 @@ class ClusterInfo:
        resources = self.spec.get_volume_resources()
        if not resources:
            resources = DEFAULT_VOLUME_RESOURCES
-        for volume_name in spec_volumes:
+        for volume_name, volume_path in spec_volumes.items():
+            # We only need to create a volume if it is fully qualified HostPath.
+            # Otherwise, we create the PVC and expect the node to allocate the volume for us.
+            if not volume_path:
+                if opts.o.debug:
+                    print(f"{volume_name} does not require an explicit PersistentVolume, since it is not a bind-mount.")
+                continue
+
            if volume_name not in named_volumes:
                if opts.o.debug:
                    print(f"{volume_name} not in pod files")
                continue
+
+            if not os.path.isabs(volume_path):
+                print(f"WARNING: {volume_name}:{volume_path} is not absolute, cannot bind volume.")
+                continue
+
+            if self.spec.is_kind_deployment():
+                host_path = client.V1HostPathVolumeSource(path=get_kind_pv_bind_mount_path(volume_name))
+            else:
+                host_path = client.V1HostPathVolumeSource(path=volume_path)
            spec = client.V1PersistentVolumeSpec(
                storage_class_name="manual",
                access_modes=["ReadWriteOnce"],
                capacity=to_k8s_resource_requirements(resources).requests,
-                host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name))
+                host_path=host_path
            )
            pv = client.V1PersistentVolume(
                metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}",
@@ -266,7 +317,7 @@ class ClusterInfo:
                merged_envs = merge_envs(
                    envs_from_compose_file(
                        service_info["environment"]), self.environment_variables.map
-                        ) if "environment" in service_info else self.environment_variables.map
+                ) if "environment" in service_info else self.environment_variables.map
                envs = envs_from_environment_variables_map(merged_envs)
                if opts.o.debug:
                    print(f"Merged envs: {envs}")
@@ -281,13 +332,37 @@ class ClusterInfo:
                    env=envs,
                    ports=[client.V1ContainerPort(container_port=port)],
                    volume_mounts=volume_mounts,
+                    security_context=client.V1SecurityContext(
+                        privileged=self.spec.get_privileged(),
+                        capabilities=client.V1Capabilities(
+                            add=self.spec.get_capabilities()
+                        ) if self.spec.get_capabilities() else None
+                    ),
                    resources=to_k8s_resource_requirements(resources),
                )
                containers.append(container)
        volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec, self.app_name)
        image_pull_secrets = [client.V1LocalObjectReference(name="laconic-registry")]
+
+        annotations = None
+        labels = {"app": self.app_name}
+
+        if self.spec.get_annotations():
+            annotations = {}
+            for key, value in self.spec.get_annotations().items():
+                for service_name in services:
+                    annotations[key.replace("{name}", service_name)] = value
+
+        if self.spec.get_labels():
+            for key, value in self.spec.get_labels().items():
+                for service_name in services:
+                    labels[key.replace("{name}", service_name)] = value
+
        template = client.V1PodTemplateSpec(
-            metadata=client.V1ObjectMeta(labels={"app": self.app_name}),
+            metadata=client.V1ObjectMeta(
+                annotations=annotations,
+                labels=labels
+            ),
            spec=client.V1PodSpec(containers=containers, image_pull_secrets=image_pull_secrets, volumes=volumes),
        )
        spec = client.V1DeploymentSpec(
@@ -20,7 +20,9 @@ from kubernetes import client, config
 from stack_orchestrator import constants
 from stack_orchestrator.deploy.deployer import Deployer, DeployerConfigGenerator
 from stack_orchestrator.deploy.k8s.helpers import create_cluster, destroy_cluster, load_images_into_kind
-from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, log_stream_from_string, generate_kind_config
+from stack_orchestrator.deploy.k8s.helpers import install_ingress_for_kind, wait_for_ingress_in_kind
+from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, containers_in_pod, log_stream_from_string
+from stack_orchestrator.deploy.k8s.helpers import generate_kind_config
 from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo
 from stack_orchestrator.opts import opts
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
@@ -88,6 +90,16 @@ class K8sDeployer(Deployer):
            if opts.o.debug:
                print(f"Sending this pv: {pv}")
            if not opts.o.dry_run:
+                try:
+                    pv_resp = self.core_api.read_persistent_volume(name=pv.metadata.name)
+                    if pv_resp:
+                        if opts.o.debug:
+                            print("PVs already present:")
+                            print(f"{pv_resp}")
+                        continue
+                except:  # noqa: E722
+                    pass
+
                pv_resp = self.core_api.create_persistent_volume(body=pv)
                if opts.o.debug:
                    print("PVs created:")
@@ -100,6 +112,17 @@ class K8sDeployer(Deployer):
                print(f"Sending this pvc: {pvc}")

            if not opts.o.dry_run:
+                try:
+                    pvc_resp = self.core_api.read_namespaced_persistent_volume_claim(
+                        name=pvc.metadata.name, namespace=self.k8s_namespace)
+                    if pvc_resp:
+                        if opts.o.debug:
+                            print("PVCs already present:")
+                            print(f"{pvc_resp}")
+                        continue
+                except:  # noqa: E722
+                    pass
+
                pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace=self.k8s_namespace)
                if opts.o.debug:
                    print("PVCs created:")
@@ -154,60 +177,80 @@ class K8sDeployer(Deployer):
                # Ensure the referenced containers are copied into kind
                load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set)
            self.connect_api()
+            if self.is_kind():
+                # Now configure an ingress controller (not installed by default in kind)
+                install_ingress_for_kind()
+                # Wait for ingress to start (deployment provisioning will fail unless this is done)
+                wait_for_ingress_in_kind()
+
        else:
            print("Dry run mode enabled, skipping k8s API connect")

        self._create_volume_data()
        self._create_deployment()

-        if not self.is_kind():
-            ingress: client.V1Ingress = self.cluster_info.get_ingress()
+        # Note: at present we don't support tls for kind (and enabling tls causes errors)
+        ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=not self.is_kind())
+        if ingress:
+            if opts.o.debug:
+                print(f"Sending this ingress: {ingress}")
+            if not opts.o.dry_run:
+                ingress_resp = self.networking_api.create_namespaced_ingress(
+                    namespace=self.k8s_namespace,
+                    body=ingress
+                )
+                if opts.o.debug:
+                    print("Ingress created:")
+                    print(f"{ingress_resp}")
+        else:
+            if opts.o.debug:
+                print("No ingress configured")

-            if ingress:
+        nodeport: client.V1Service = self.cluster_info.get_nodeport()
+        if nodeport:
+            if opts.o.debug:
+                print(f"Sending this nodeport: {nodeport}")
+            if not opts.o.dry_run:
+                nodeport_resp = self.core_api.create_namespaced_service(
+                    namespace=self.k8s_namespace,
+                    body=nodeport
+                )
                if opts.o.debug:
-                    print(f"Sending this ingress: {ingress}")
-                if not opts.o.dry_run:
-                    ingress_resp = self.networking_api.create_namespaced_ingress(
-                        namespace=self.k8s_namespace,
-                        body=ingress
-                    )
-                    if opts.o.debug:
-                        print("Ingress created:")
-                        print(f"{ingress_resp}")
-            else:
-                if opts.o.debug:
-                    print("No ingress configured")
+                    print("NodePort created:")
+                    print(f"{nodeport_resp}")

    def down(self, timeout, volumes):  # noqa: C901
        self.connect_api()
        # Delete the k8s objects
-        # Create the host-path-mounted PVs for this deployment
-        pvs = self.cluster_info.get_pvs()
-        for pv in pvs:
-            if opts.o.debug:
-                print(f"Deleting this pv: {pv}")
-            try:
-                pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name)
-                if opts.o.debug:
-                    print("PV deleted:")
-                    print(f"{pv_resp}")
-            except client.exceptions.ApiException as e:
-                _check_delete_exception(e)

-        # Figure out the PVCs for this deployment
-        pvcs = self.cluster_info.get_pvcs()
-        for pvc in pvcs:
-            if opts.o.debug:
-                print(f"Deleting this pvc: {pvc}")
-            try:
-                pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim(
-                    name=pvc.metadata.name, namespace=self.k8s_namespace
-                )
+        if volumes:
+            # Create the host-path-mounted PVs for this deployment
+            pvs = self.cluster_info.get_pvs()
+            for pv in pvs:
                if opts.o.debug:
-                    print("PVCs deleted:")
-                    print(f"{pvc_resp}")
-            except client.exceptions.ApiException as e:
-                _check_delete_exception(e)
+                    print(f"Deleting this pv: {pv}")
+                try:
+                    pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name)
+                    if opts.o.debug:
+                        print("PV deleted:")
+                        print(f"{pv_resp}")
+                except client.exceptions.ApiException as e:
+                    _check_delete_exception(e)
+
+            # Figure out the PVCs for this deployment
+            pvcs = self.cluster_info.get_pvcs()
+            for pvc in pvcs:
+                if opts.o.debug:
+                    print(f"Deleting this pvc: {pvc}")
+                try:
+                    pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim(
+                        name=pvc.metadata.name, namespace=self.k8s_namespace
+                    )
+                    if opts.o.debug:
+                        print("PVCs deleted:")
+                        print(f"{pvc_resp}")
+                except client.exceptions.ApiException as e:
+                    _check_delete_exception(e)

        # Figure out the ConfigMaps for this deployment
        cfg_maps = self.cluster_info.get_configmaps()
@@ -245,20 +288,34 @@ class K8sDeployer(Deployer):
        except client.exceptions.ApiException as e:
            _check_delete_exception(e)

-        if not self.is_kind():
-            ingress: client.V1Ingress = self.cluster_info.get_ingress()
-            if ingress:
-                if opts.o.debug:
-                    print(f"Deleting this ingress: {ingress}")
-                try:
-                    self.networking_api.delete_namespaced_ingress(
-                        name=ingress.metadata.name, namespace=self.k8s_namespace
-                    )
-                except client.exceptions.ApiException as e:
-                    _check_delete_exception(e)
-            else:
-                if opts.o.debug:
-                    print("No ingress to delete")
+        ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=not self.is_kind())
+        if ingress:
+            if opts.o.debug:
+                print(f"Deleting this ingress: {ingress}")
+            try:
+                self.networking_api.delete_namespaced_ingress(
+                    name=ingress.metadata.name, namespace=self.k8s_namespace
+                )
+            except client.exceptions.ApiException as e:
+                _check_delete_exception(e)
+        else:
+            if opts.o.debug:
+                print("No ingress to delete")
+
+        nodeport: client.V1Service = self.cluster_info.get_nodeport()
+        if nodeport:
+            if opts.o.debug:
+                print(f"Deleting this nodeport: {ingress}")
+            try:
+                self.core_api.delete_namespaced_service(
+                    namespace=self.k8s_namespace,
+                    name=nodeport.metadata.name
+                )
+            except client.exceptions.ApiException as e:
+                _check_delete_exception(e)
+        else:
+            if opts.o.debug:
+                print("No nodeport to delete")

        if self.is_kind():
            # Destroy the kind cluster
@@ -359,9 +416,15 @@ class K8sDeployer(Deployer):
            log_data = "******* Pods not running ********\n"
        else:
            k8s_pod_name = pods[0]
+            containers = containers_in_pod(self.core_api, k8s_pod_name)
            # If the pod is not yet started, the logs request below will throw an exception
            try:
-                log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
+                log_data = ""
+                for container in containers:
+                    container_log = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container=container)
+                    container_log_lines = container_log.splitlines()
+                    for line in container_log_lines:
+                        log_data += f"{container}: {line}\n"
            except client.exceptions.ApiException as e:
                if opts.o.debug:
                    print(f"Error from read_namespaced_pod_log: {e}")
@@ -13,13 +13,14 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.

-from kubernetes import client
+from kubernetes import client, utils, watch
 import os
 from pathlib import Path
 import subprocess
 import re
 from typing import Set, Mapping, List

+from stack_orchestrator.util import get_k8s_dir, error_exit
 from stack_orchestrator.opts import opts
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names
 from stack_orchestrator.deploy.deployer import DeployerException
@@ -44,6 +45,33 @@ def destroy_cluster(name: str):
    _run_command(f"kind delete cluster --name {name}")


+def wait_for_ingress_in_kind():
+    core_v1 = client.CoreV1Api()
+    for i in range(20):
+        warned_waiting = False
+        w = watch.Watch()
+        for event in w.stream(func=core_v1.list_namespaced_pod,
+                              namespace="ingress-nginx",
+                              label_selector="app.kubernetes.io/component=controller",
+                              timeout_seconds=30):
+            if event['object'].status.container_statuses:
+                if event['object'].status.container_statuses[0].ready is True:
+                    if warned_waiting:
+                        print("Ingress controller is ready")
+                    return
+            print("Waiting for ingress controller to become ready...")
+            warned_waiting = True
+    error_exit("ERROR: Timed out waiting for ingress to become ready")
+
+
+def install_ingress_for_kind():
+    api_client = client.ApiClient()
+    ingress_install = os.path.abspath(get_k8s_dir().joinpath("components", "ingress", "ingress-nginx-kind-deploy.yaml"))
+    if opts.o.debug:
+        print("Installing nginx ingress controller in kind cluster")
+    utils.create_from_yaml(api_client, yaml_file=ingress_install)
+
+
 def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]):
    for image in image_set:
        result = _run_command(f"kind load docker-image {image} --name {kind_cluster_name}")
@@ -62,6 +90,17 @@ def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
    return pods


+def containers_in_pod(core_api: client.CoreV1Api, pod_name: str):
+    containers = []
+    pod_response = core_api.read_namespaced_pod(pod_name, namespace="default")
+    if opts.o.debug:
+        print(f"pod_response: {pod_response}")
+    pod_containers = pod_response.spec.containers
+    for pod_container in pod_containers:
+        containers.append(pod_container.name)
+    return containers
+
+
 def log_stream_from_string(s: str):
    # Note response has to be UTF-8 encoded because the caller expects to decode it
    yield ("ignore", s.encode())
@@ -81,7 +120,7 @@ def named_volumes_from_pod_files(parsed_pod_files):
    return named_volumes


-def get_node_pv_mount_path(volume_name: str):
+def get_kind_pv_bind_mount_path(volume_name: str):
    return f"/mnt/{volume_name}"


@@ -106,11 +145,14 @@ def volume_mounts_for_service(parsed_pod_files, service):
                            mount_path = mount_split[1]
                            mount_options = mount_split[2] if len(mount_split) == 3 else None
                            if opts.o.debug:
-                                print(f"volumne_name: {volume_name}")
+                                print(f"volume_name: {volume_name}")
                                print(f"mount path: {mount_path}")
                                print(f"mount options: {mount_options}")
                            volume_device = client.V1VolumeMount(
-                                mount_path=mount_path, name=volume_name, read_only="ro" == mount_options)
+                                mount_path=mount_path,
+                                name=volume_name,
+                                read_only="ro" == mount_options
+                            )
                            result.append(volume_device)
    return result

@@ -133,18 +175,8 @@ def volumes_for_pod_files(parsed_pod_files, spec, app_name):
    return result


-def _get_host_paths_for_volumes(parsed_pod_files):
-    result = {}
-    for pod in parsed_pod_files:
-        parsed_pod_file = parsed_pod_files[pod]
-        if "volumes" in parsed_pod_file:
-            volumes = parsed_pod_file["volumes"]
-            for volume_name in volumes.keys():
-                volume_definition = volumes[volume_name]
-                if volume_definition and "driver_opts" in volume_definition:
-                    host_path = volume_definition["driver_opts"]["device"]
-                    result[volume_name] = host_path
-    return result
+def _get_host_paths_for_volumes(deployment_context):
+    return deployment_context.spec.get_volumes()


 def _make_absolute_host_path(data_mount_path: Path, deployment_dir: Path) -> Path:
@@ -152,12 +184,12 @@ def _make_absolute_host_path(data_mount_path: Path, deployment_dir: Path) -> Pat
        return data_mount_path
    else:
        # Python Path voodo that looks pretty odd:
-        return Path.cwd().joinpath(deployment_dir.joinpath("compose").joinpath(data_mount_path)).resolve()
+        return Path.cwd().joinpath(deployment_dir.joinpath(data_mount_path)).resolve()


 def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context):
    volume_definitions = []
-    volume_host_path_map = _get_host_paths_for_volumes(parsed_pod_files)
+    volume_host_path_map = _get_host_paths_for_volumes(deployment_context)
    # Note these paths are relative to the location of the pod files (at present)
    # So we need to fix up to make them correct and absolute because kind assumes
    # relative to the cwd.
@@ -177,14 +209,15 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context):
                        volume_name = mount_split[0]
                        mount_path = mount_split[1]
                        if opts.o.debug:
-                            print(f"volumne_name: {volume_name}")
+                            print(f"volume_name: {volume_name}")
                            print(f"map: {volume_host_path_map}")
                            print(f"mount path: {mount_path}")
                        if volume_name not in deployment_context.spec.get_configmaps():
-                            volume_definitions.append(
-                                f"  - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n"
-                                f"    containerPath: {get_node_pv_mount_path(volume_name)}\n"
-                            )
+                            if volume_host_path_map[volume_name]:
+                                volume_definitions.append(
+                                    f"  - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n"
+                                    f"    containerPath: {get_kind_pv_bind_mount_path(volume_name)}\n"
+                                )
    return (
        "" if len(volume_definitions) == 0 else (
            "  extraMounts:\n"
@@ -193,7 +226,8 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context):
    )


-def _generate_kind_port_mappings(parsed_pod_files):
+# TODO: decide if we need this functionality
+def _generate_kind_port_mappings_from_services(parsed_pod_files):
    port_definitions = []
    for pod in parsed_pod_files:
        parsed_pod_file = parsed_pod_files[pod]
@@ -215,6 +249,19 @@ def _generate_kind_port_mappings(parsed_pod_files):
    )


+def _generate_kind_port_mappings(parsed_pod_files):
+    port_definitions = []
+    # For now we just map port 80 for the nginx ingress controller we install in kind
+    port_string = "80"
+    port_definitions.append(f"  - containerPort: {port_string}\n    hostPort: {port_string}\n")
+    return (
+        "" if len(port_definitions) == 0 else (
+            "  extraPortMappings:\n"
+            f"{''.join(port_definitions)}"
+        )
+    )
+
+
 # Note: this makes any duplicate definition in b overwrite a
 def merge_envs(a: Mapping[str, str], b: Mapping[str, str]) -> Mapping[str, str]:
    result = {**a, **b}
@@ -279,6 +326,12 @@ def generate_kind_config(deployment_dir: Path, deployment_context):
        "apiVersion: kind.x-k8s.io/v1alpha4\n"
        "nodes:\n"
        "- role: control-plane\n"
+        "  kubeadmConfigPatches:\n"
+        "    - |\n"
+        "      kind: InitConfiguration\n"
+        "      nodeRegistration:\n"
+        "        kubeletExtraArgs:\n"
+        "          node-labels: \"ingress-ready=true\"\n"
        f"{port_mappings_yml}\n"
        f"{mounts_yml}\n"
    )
@@ -72,8 +72,18 @@ class Spec:
    obj: typing.Any
    file_path: Path

-    def __init__(self) -> None:
-        pass
+    def __init__(self, file_path: Path = None, obj={}) -> None:
+        self.file_path = file_path
+        self.obj = obj
+
+    def __getitem__(self, item):
+        return self.obj[item]
+
+    def __contains__(self, item):
+        return item in self.obj
+
+    def get(self, item, default=None):
+        return self.obj.get(item, default)

    def init_from_file(self, file_path: Path):
        with file_path:
@@ -81,8 +91,8 @@ class Spec:
            self.file_path = file_path

    def get_image_registry(self):
-        return (self.obj[constants.image_resigtry_key]
-                if self.obj and constants.image_resigtry_key in self.obj
+        return (self.obj[constants.image_registry_key]
+                if self.obj and constants.image_registry_key in self.obj
                else None)

    def get_volumes(self):
@@ -106,3 +116,27 @@ class Spec:
                if self.obj and constants.network_key in self.obj
                and constants.http_proxy_key in self.obj[constants.network_key]
                else None)
+
+    def get_annotations(self):
+        return self.obj.get("annotations", {})
+
+    def get_labels(self):
+        return self.obj.get("labels", {})
+
+    def get_privileged(self):
+        return "true" == str(self.obj.get("security", {}).get("privileged", "false")).lower()
+
+    def get_capabilities(self):
+        return self.obj.get("security", {}).get("capabilities", [])
+
+    def get_deployment_type(self):
+        return self.obj[constants.deploy_to_key]
+
+    def is_kubernetes_deployment(self):
+        return self.get_deployment_type() in [constants.k8s_kind_deploy_type, constants.k8s_deploy_type]
+
+    def is_kind_deployment(self):
+        return self.get_deployment_type() in [constants.k8s_kind_deploy_type]
+
+    def is_docker_deployment(self):
+        return self.get_deployment_type() in [constants.compose_deploy_type]
@@ -44,7 +44,7 @@ def _fixup_url_spec(spec_file_name: str, url: str):
    - host-name: {parsed_url.hostname}
      routes:
        - path: '{parsed_url.path if parsed_url.path else "/"}'
-          proxy-to: webapp:3000
+          proxy-to: webapp:80
    '''
    spec_file_path = Path(spec_file_name)
    with open(spec_file_path) as rfile:
@@ -147,6 +147,7 @@ def command(ctx, laconic_config, deployment_parent_dir,

    previous_requests = load_known_requests(state_file)
    requests.sort(key=lambda r: r.createTime)
+    requests.reverse()

    # Find deployments.
    deployments = {}
@@ -355,11 +355,13 @@ def generate_hostname_for_app(app):

 def skip_by_tag(r, include_tags, exclude_tags):
    for tag in exclude_tags:
-        if tag and r.attributes.tags and tag in r.attributes.tags:
+        if r.attributes.tags and tag in r.attributes.tags:
            return True

-    for tag in include_tags:
-        if tag and (not r.attributes.tags or tag not in r.attributes.tags):
-            return True
+    if include_tags:
+        for tag in include_tags:
+            if r.attributes.tags and tag in r.attributes.tags:
+                return False
+        return True

    return False
@@ -26,7 +26,7 @@ import importlib.resources
 from pathlib import Path
 import yaml
 from stack_orchestrator.constants import stack_file_name
-from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
+from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit, warn_exit


 class GitProgress(git.RemoteProgress):
@@ -249,8 +249,8 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches
            error_exit(f"stack {stack} does not exist")
        with stack_file_path:
            stack_config = yaml.safe_load(open(stack_file_path, "r"))
-            if "repos" not in stack_config:
-                error_exit(f"stack {stack} does not define any repositories")
+            if "repos" not in stack_config or stack_config["repos"] is None:
+                warn_exit(f"stack {stack} does not define any repositories")
            else:
                repos_in_scope = stack_config["repos"]
    else:
@@ -146,6 +146,12 @@ def get_config_file_dir():
    return source_config_dir


+def get_k8s_dir():
+    data_dir = Path(__file__).absolute().parent.joinpath("data")
+    source_config_dir = data_dir.joinpath("k8s")
+    return source_config_dir
+
+
 def get_parsed_deployment_spec(spec_file):
    spec_file_path = Path(spec_file)
    try:
@@ -189,5 +195,10 @@ def error_exit(s):
    sys.exit(1)


+def warn_exit(s):
+    print(f"WARN: {s}")
+    sys.exit(0)
+
+
 def env_var_map_from_file(file: Path) -> Mapping[str, str]:
    return dotenv_values(file)
@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+    # Dump environment variables for debugging
+    echo "Environment variables:"
+    env
+fi
+
+if [ "$1" == "from-path" ]; then
+    TEST_TARGET_SO="laconic-so"
+else
+    TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+fi
+
+stack="test-database"
+spec_file=${stack}-spec.yml
+deployment_dir=${stack}-deployment
+
+# Helper functions: TODO move into a separate file
+wait_for_pods_started () {
+    for i in {1..50}
+    do
+        local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps )
+
+        if [[ "$ps_output" == *"Running containers:"* ]]; then
+            # if ready, return
+            return
+        else
+            # if not ready, wait
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "waiting for pods to start: FAILED"
+    delete_cluster_exit
+}
+
+wait_for_test_complete () {
+    for i in {1..50}
+    do
+
+        local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+
+    if [[ "${log_output}" == *"Database test client: test complete"* ]]; then
+            # if ready, return
+            return
+        else
+            # if not ready, wait
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "waiting for test complete: FAILED"
+    delete_cluster_exit
+}
+
+
+delete_cluster_exit () {
+    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+    exit 1
+}
+
+# Set a non-default repo dir
+export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
+echo "Testing this package: $TEST_TARGET_SO"
+echo "Test version command"
+reported_version_string=$( $TEST_TARGET_SO version )
+echo "Version reported is: ${reported_version_string}"
+echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
+rm -rf $CERC_REPO_BASE_DIR
+mkdir -p $CERC_REPO_BASE_DIR
+$TEST_TARGET_SO --stack ${stack} setup-repositories
+$TEST_TARGET_SO --stack ${stack} build-containers
+# Test basic stack-orchestrator deploy to k8s
+test_deployment_dir=$CERC_REPO_BASE_DIR/${deployment_dir}
+test_deployment_spec=$CERC_REPO_BASE_DIR/${spec_file}
+
+$TEST_TARGET_SO --stack ${stack} deploy --deploy-to k8s-kind init --output $test_deployment_spec
+# Check the file now exists
+if [ ! -f "$test_deployment_spec" ]; then
+    echo "deploy init test: spec file not present"
+    echo "deploy init test: FAILED"
+    exit 1
+fi
+echo "deploy init test: passed"
+
+# Switch to a full path for the data dir so it gets provisioned as a host bind mounted volume and preserved beyond cluster lifetime
+sed -i "s|^\(\s*db-data:$\)$|\1 ${test_deployment_dir}/data/db-data|" $test_deployment_spec
+
+$TEST_TARGET_SO --stack ${stack} deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
+# Check the deployment dir exists
+if [ ! -d "$test_deployment_dir" ]; then
+    echo "deploy create test: deployment directory not present"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+echo "deploy create test: passed"
+
+# Try to start the deployment
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+wait_for_pods_started
+# Check logs command works
+wait_for_test_complete
+log_output_1=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_1" == *"Database test client: test data does not exist"* ]]; then
+    echo "Create database content test: passed"
+else
+    echo "Create database content test: FAILED"
+    delete_cluster_exit
+fi
+
+# Stop then start again and check the volume was preserved
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
+# Sleep a bit just in case
+sleep 20
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+wait_for_pods_started
+wait_for_test_complete
+
+log_output_2=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_2" == *"Database test client: test data already exists"* ]]; then
+    echo "Retain database content test: passed"
+else
+    echo "Retain database content test: FAILED"
+    delete_cluster_exit
+fi
+
+# Stop and clean up
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+echo "Test passed"
@@ -63,7 +63,7 @@ $TEST_TARGET_SO --stack test deploy down
 # The next time we bring the container up the volume will be old (from the previous run above)
 $TEST_TARGET_SO --stack test deploy up
 log_output_1=$( $TEST_TARGET_SO --stack test deploy logs )
-if [[ "$log_output_1" == *"Filesystem is old"* ]]; then
+if [[ "$log_output_1" == *"filesystem is old"* ]]; then
    echo "Retain volumes test: passed"
 else
    echo "Retain volumes test: FAILED"
@@ -73,7 +73,7 @@ $TEST_TARGET_SO --stack test deploy down --delete-volumes
 # Now when we bring the container up the volume will be new again
 $TEST_TARGET_SO --stack test deploy up
 log_output_2=$( $TEST_TARGET_SO --stack test deploy logs )
-if [[ "$log_output_2" == *"Filesystem is fresh"* ]]; then
+if [[ "$log_output_2" == *"filesystem is fresh"* ]]; then
    echo "Delete volumes test: passed"
 else
    echo "Delete volumes test: FAILED"
@@ -121,7 +121,7 @@ echo "deploy create output file test: passed"
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
 # Check logs command works
 log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then
+if [[ "$log_output_3" == *"filesystem is fresh"* ]]; then
    echo "deployment logs test: passed"
 else
    echo "deployment logs test: FAILED"
@@ -158,7 +158,7 @@ $TEST_TARGET_SO deployment --dir $test_deployment_dir stop
 sleep 20
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
 log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_5" == *"Filesystem is old"* ]]; then
+if [[ "$log_output_5" == *"filesystem is old"* ]]; then
    echo "Retain volumes test: passed"
 else
    echo "Retain volumes test: FAILED"
@@ -76,6 +76,10 @@ if [ ! -f "$test_deployment_spec" ]; then
    exit 1
 fi
 echo "deploy init test: passed"
+
+# Switch to a full path for bind mount.
+sed -i "s|^\(\s*test-data-bind:$\)$|\1 ${test_deployment_dir}/data/test-data-bind|" $test_deployment_spec
+
 $TEST_TARGET_SO --stack test deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
 # Check the deployment dir exists
 if [ ! -d "$test_deployment_dir" ]; then
@@ -99,7 +103,7 @@ if [ ! "$create_file_content" == "create-command-output-data"  ]; then
 fi

 # Add a config file to be picked up by the ConfigMap before starting.
-echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/data/test-config/test_config
+echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/configmaps/test-config/test_config

 echo "deploy create output file test: passed"
 # Try to start the deployment
@@ -107,11 +111,13 @@ $TEST_TARGET_SO deployment --dir $test_deployment_dir start
 wait_for_pods_started
 # Check logs command works
 wait_for_log_output
+sleep 1
 log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then
+if [[ "$log_output_3" == *"filesystem is fresh"* ]]; then
    echo "deployment logs test: passed"
 else
    echo "deployment logs test: FAILED"
+    echo $log_output_3
    delete_cluster_exit
 fi

@@ -140,6 +146,26 @@ else
    delete_cluster_exit
 fi

+# Check that the bind-mount volume is mounted.
+log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_5" == *"/data: MOUNTED"* ]]; then
+    echo "deployment bind volumes test: passed"
+else
+    echo "deployment bind volumes test: FAILED"
+    echo $log_output_5
+    delete_cluster_exit
+fi
+
+# Check that the provisioner managed volume is mounted.
+log_output_6=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_6" == *"/data2: MOUNTED"* ]]; then
+    echo "deployment provisioner volumes test: passed"
+else
+    echo "deployment provisioner volumes test: FAILED"
+    echo $log_output_6
+    delete_cluster_exit
+fi
+
 # Stop then start again and check the volume was preserved
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop
 # Sleep a bit just in case
@@ -148,13 +174,26 @@ sleep 20
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
 wait_for_pods_started
 wait_for_log_output
-log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_5" == *"Filesystem is old"* ]]; then
-    echo "Retain volumes test: passed"
+sleep 1
+
+log_output_10=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_10" == *"/data filesystem is old"* ]]; then
+    echo "Retain bind volumes test: passed"
 else
-    echo "Retain volumes test: FAILED"
+    echo "Retain bind volumes test: FAILED"
    delete_cluster_exit
 fi
+
+# These volumes will be completely destroyed by the kind delete/create, because they lived inside
+# the kind container.  So, unlike the bind-mount case, they will appear fresh after the restart.
+log_output_11=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_11" == *"/data2 filesystem is fresh"* ]]; then
+    echo "Fresh provisioner volumes test: passed"
+else
+    echo "Fresh provisioner volumes test: FAILED"
+    delete_cluster_exit
+fi
+
 # Stop and clean up
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
 echo "Test passed"
Author	SHA1	Message	Date
dboreham	e22792dc4b	Merge branch 'main' into ci-test	2024-02-17 07:30:29 -07:00
dboreham	0b0394a940	Use absolute path for the data volume (#749 ) Reviewed-on: cerc-io/stack-orchestrator#749 Co-authored-by: David Boreham <david@bozemanpass.com> Co-committed-by: David Boreham <david@bozemanpass.com>	2024-02-17 14:29:53 +00:00
dboreham	d79053b137	Merge branch 'main' into ci-test	2024-02-17 07:24:12 -07:00
dboreham	37b9500483	Support non-tls ingress for kind (#748 ) Reviewed-on: cerc-io/stack-orchestrator#748 Co-authored-by: David Boreham <david@bozemanpass.com> Co-committed-by: David Boreham <david@bozemanpass.com>	2024-02-17 01:54:30 +00:00
telackey	3c3e582939	Minor envsubst improvements. (#746 ) Minor fixes to envsubst for webapps. Somewhat specially treated is `LACONIC_HOSTED_CONFIG_homepage` which can be used to replace the homepage in package.json. With react, this gets an extra `/` though, which we need to remove. Reviewed-on: cerc-io/stack-orchestrator#746 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-02-16 04:11:09 +00:00
dboreham	85204ca29a	Enable debug output on test	2024-02-15 08:41:54 -07:00
dboreham	d6d016cad8	Merge branch 'main' into ci-test	2024-02-15 08:13:10 -07:00
dboreham	26d265360d	Rename workflow file	2024-02-15 07:39:04 -07:00
dboreham	f81b78cfbc	Update .gitea/workflows/triggers/test-database	2024-02-15 14:35:56 +00:00
dboreham	d9bb6b3588	Test Database Stack (#737 ) Reviewed-on: cerc-io/stack-orchestrator#737	2024-02-15 05:26:29 +00:00
dboreham	b59beb66eb	Add simple quick deploy script (#743 ) Reviewed-on: cerc-io/stack-orchestrator#743 Co-authored-by: David Boreham <david@bozemanpass.com> Co-committed-by: David Boreham <david@bozemanpass.com>	2024-02-15 05:00:51 +00:00
telackey	65d67dba10	Fix k8s and enable it by default on PRs (#742 ) Reviewed-on: cerc-io/stack-orchestrator#742 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-02-14 23:50:09 +00:00
telackey	b22c72e715	For k8s, use provisioner-managed volumes when an absolute host path is not specified. (#741 ) In kind, when we bind-mount a host directory it is first mounted into the kind container at /mnt, then into the pod at the desired location. We accidentally picked this up for full-blown k8s, and were creating volumes at /mnt. This changes the behavior for both kind and regular k8s so that bind mounts are only allowed if a fully-qualified path is specified. If no path is specified at all, a default storageClass is assumed to be present, and the volume managed by a provisioner. Eg, for kind, the default provisioner is: https://github.com/rancher/local-path-provisioner ``` stack: test deploy-to: k8s-kind config: test-variable-1: test-value-1 network: ports: test: - '80' volumes: # this will be bind-mounted to a host-path test-data-bind: /srv/data # this will be managed by the k8s node test-data-auto: configmaps: test-config: ./configmap/test-config ``` Reviewed-on: cerc-io/stack-orchestrator#741 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-02-14 21:45:01 +00:00
telackey	c9444591f5	Fix default webapp port number. (#740 ) Reviewed-on: cerc-io/stack-orchestrator#740 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-02-09 01:20:41 +00:00
telackey	903f3b10e2	Add support for annotations and labels in spec. (#739 ) ``` stack: webapp-deployer-backend deploy-to: k8s annotations: foo.bar.annot/{name}: baz labels: a.b.c/{name}.blah: "value" ``` Reviewed-on: cerc-io/stack-orchestrator#739 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-02-09 00:11:07 +00:00
telackey	72ed2eb91a	Fix bad test in tag check. (#738 ) Reviewed-on: cerc-io/stack-orchestrator#738 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-02-08 20:38:41 +00:00
dboreham	3b553bb660	Revert to the stock runner	2024-01-30 11:13:21 -07:00
dboreham	466c149f36	Switch back to alternative runner	2024-01-30 07:39:05 -07:00
dboreham	3c84ebff68	Switch back to stock runner	2024-01-30 07:01:13 -07:00
dboreham	d5244726aa	Use new runner	2024-01-28 16:25:45 -07:00
dboreham	15ae2c0cdf	Merge main	2024-01-28 16:24:26 -07:00
dboreham	6d5f661750	Try a longer sleep	2024-01-21 17:42:09 -07:00
dboreham	bc872b09fb	Add output to check cgroups version	2024-01-21 17:23:28 -07:00
dboreham	a751e87dd4	Switch back to stock runner	2024-01-21 17:21:35 -07:00
dboreham	5ba9a803de	Trigger k8s deploy test off changes to its scripts	2024-01-21 17:19:39 -07:00
dboreham	b56c84b93a	Merge main	2024-01-21 17:16:09 -07:00
dboreham	1520f7413d	Debug CI	2023-11-08 20:02:21 -07:00
dboreham	1d78430679	Merge branch 'main' into ci-test	2023-11-08 20:01:26 -07:00
dboreham	46b726be9b	Merge branch 'main' into ci-test	2023-10-12 15:26:38 -06:00
dboreham	b3b57b6f28	Merge branch 'main' into ci-test	2023-10-03 07:25:32 -06:00
dboreham	69b352dcce	Make the block window wider	2023-08-17 08:31:21 -06:00
dboreham	4e656df709	Extend the wait	2023-08-17 07:23:33 -06:00
dboreham	7fc6375886	Revert long wait	2023-08-17 03:04:22 -06:00
dboreham	ac02753ffe	Build lighthouse for older CPUs	2023-08-17 02:09:29 -06:00
dboreham	69d59b109b	Add stack logs in case of fail	2023-08-17 00:36:36 -06:00
dboreham	f62a8ff54f	Try waiting even longer	2023-08-16 23:41:39 -06:00
dboreham	82a839d15f	Try waiting longer for a new block	2023-08-16 22:12:45 -06:00
dboreham	5a31cb50c3	Add log output at end of test	2023-08-16 20:57:59 -06:00
dboreham	fae2438e97	Comment out stack status step	2023-08-16 15:45:19 -06:00
dboreham	952ea29ca5	Add time stamps to test script	2023-08-16 13:50:27 -06:00
dboreham	fbfc162ab7	Merge branch 'main' into ci-test	2023-08-16 09:32:29 -06:00
dboreham	6ff81b2049	Merge branch 'main' into ci-test	2023-08-03 14:30:10 -06:00
dboreham	3b3c688ee9	Merge branch 'dboreham/update-lighthouse' into ci-test	2023-07-27 20:14:54 -06:00
dboreham	16bef78312	Merge branch 'main' into ci-test	2023-07-27 20:14:45 -06:00
dboreham	12e68ff1ec	Use latest stable lighthouse release	2023-07-27 20:14:12 -06:00
dboreham	6f3f20739d	Merge branch 'main' into ci-test	2023-07-19 20:53:16 -06:00
dboreham	caabaa35ff	Merge branch 'main' into ci-test	2023-07-19 20:38:33 -06:00
dboreham	a265794bd7	Update fixturenet-eth-test.yml Change to trigger CI on Gitea	2023-07-18 11:43:18 -06:00
dboreham	5d7e7294c9	Update fixturenet-eth-test.yml Change to trigger CI in gitea	2023-07-18 11:33:01 -06:00
dboreham	d9805666c3	Merge main	2023-06-29 15:16:38 -06:00
dboreham	582b92d277	Merge main	2023-06-29 15:14:40 -06:00
David Boreham	dc19d68aa4	Merge branch 'main' into ci-test	2023-06-05 20:50:43 -06:00
David Boreham	d57e8d0efd	Merge branch 'dboreham/build-local-lcli' into ci-test	2023-06-05 16:46:50 -06:00
telackey	91d3613247	Include --eth1-block-hash	2023-06-05 15:45:07 -05:00
David Boreham	76ab4a7717	Merge branch 'dboreham/build-local-lcli' into ci-test	2023-06-04 23:04:40 -06:00
David Boreham	fd40aedadd	Add lcli container to fixturenet-eth stack	2023-06-04 23:03:48 -06:00
David Boreham	3748fa38b1	Update ldcli options	2023-06-04 23:01:43 -06:00
David Boreham	aa4c52c174	Fix typo	2023-06-04 16:43:38 -06:00
David Boreham	609128d004	Blind fix for lcli error	2023-06-04 16:42:44 -06:00
David Boreham	47164cb5f5	Test updated lcli command	2023-06-04 16:12:16 -06:00
David Boreham	93c47ec5cc	Merge branch 'dboreham/build-local-lcli' into ci-test	2023-06-04 15:41:24 -06:00
David Boreham	700ce586a6	Merge main	2023-06-04 15:41:01 -06:00
David Boreham	223aacb16f	Enable portable lcli build	2023-06-04 14:30:01 -06:00
David Boreham	c120ee2086	Pull lighthouse repo	2023-06-04 14:29:07 -06:00
David Boreham	685e11ed6f	Build lcli locally	2023-06-04 14:27:07 -06:00
dboreham	6cf9869a57	Merge branch 'dboreham/add-foundry-to-plugeth-stack' into ci-test	2023-05-30 23:22:03 -06:00
dboreham	00d34944a2	Add foundry to fixturenet-plugeth-tx	2023-05-30 23:20:26 -06:00
dboreham	40424cc3b6	Merge branch 'main' into ci-test	2023-05-30 22:33:37 -06:00
dboreham	b594d15e91	Trigger CI	2023-05-30 12:23:48 -06:00
dboreham	500e60c480	Trigger CI	2023-05-30 08:49:36 -06:00
dboreham	8c3a8eb33d	Add fixturenet-plugeth test	2023-05-25 16:56:59 -06:00
dboreham	27067e061c	Merge branch 'main' into ci-test	2023-05-25 16:46:36 -06:00
dboreham	eab5a6ccf1	Trigger CI	2023-05-24 21:49:35 -06:00
dboreham	7ed2448f03	Merge current main Former-commit-id: d82ba6ed5035203d1ce3846744309210caadda62	2023-05-18 13:57:35 -06:00
dboreham	e9e1d8bcd4	Merge branch 'main' into ci-test Former-commit-id: 966539faa46bf33a3b6471a3289a4fb7c4685cc7	2023-05-18 13:55:29 -06:00
dboreham	0563764ca4	Trigger CI Former-commit-id: bb1d81a906000d5acf22269eaca53c6ab32c5605	2023-05-02 14:22:40 -06:00
dboreham	069dd50fd3	Merge branch 'main' into ci-test Former-commit-id: 4eb970350bbfca04ea98b21a6aaf0acfa3daa8cd	2023-04-28 09:44:05 -06:00
dboreham	595fbe4eca	Force CI again Former-commit-id: 401ab334f00273307f64397f06f2b295c1151085	2023-04-20 20:17:19 -06:00
dboreham	3e949b403f	Force CI again Former-commit-id: fa822a71297ad39d5f8301321e7230f55bb0cb27	2023-04-20 19:06:20 -06:00
dboreham	5524e95434	Force CI to run Former-commit-id: 99ecc01e52eda6bf162679bc0dda79c0438b14cb	2023-04-20 18:24:45 -06:00
dboreham	616982d9ba	Merge branch 'main' into ci-test Former-commit-id: b3b24e3f1870763025d9ac9b07685b1a85567874	2023-04-20 17:40:41 -06:00
dboreham	2cbb6e1f1d	Add debug output Former-commit-id: 27d23144e4003fa131a7fc95b7f527848b6181ad	2023-04-20 15:00:20 -06:00
dboreham	0712012cf4	Start docker inside the executor Former-commit-id: 2948691fc712898c4ad04bbf11296f8ecbbd5b0b	2023-04-20 13:58:20 -06:00