unused

This saves about 1GB of space in the image.

Reviewed-on: cerc-io/stack-orchestrator#773
Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com>
Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>

.gitea/workflows/fixturenet-eth-plugeth-arm-test.yml

@@ -0,0 +1,45 @@
+name: Fixturenet-Eth-Plugeth-Arm-Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-arm-test'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '2 14 * * *'
+
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
+
+jobs:
+  test:
+    name: "Run an Ethereum plugeth fixturenet test"
+    runs-on: ubuntu-latest-arm
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Run fixturenet-eth tests"
+        run: ./tests/fixturenet-eth-plugeth/run-test.sh

.gitea/workflows/lint.yml

@@ -0,0 +1,21 @@
+name: Lint Checks
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches: '*'
+
+jobs:
+  test:
+    name: "Run linter"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name : "Run flake8"
+        uses: py-actions/flake8@v2

.gitea/workflows/test-container-registry.yml

@@ -0,0 +1,54 @@
+name: Container Registry Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-container-registry'
+      - '.gitea/workflows/test-container-registry.yml'
+      - 'tests/container-registry/run-test.sh'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '6 19 * * *'
+
+jobs:
+  test:
+    name: "Run contaier registry hosting test on kind/k8s"
+    runs-on: ubuntu-22.04
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Install ed" # Only needed until we remove the need to edit the spec file
+        run: apt update && apt install -y ed
+      - name: "Run container registry deployment test"
+        run: |
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/container-registry/run-test.sh
+

.gitea/workflows/test-database.yml

@@ -0,0 +1,52 @@
+name: Database Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-database'
+      - '.gitea/workflows/test-database.yml'
+      - 'tests/database/run-test.sh'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '5 18 * * *'
+
+jobs:
+  test:
+    name: "Run database hosting test on kind/k8s"
+    runs-on: ubuntu-22.04
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Run database deployment test"
+        run: |
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/database/run-test.sh
+

.gitea/workflows/test-k8s-deploy.yml

@@ -1,6 +1,8 @@
 name: K8s Deploy Test
 
 on:
+  pull_request:
+    branches: '*'
   push:
     branches: '*'
     paths:

.gitea/workflows/triggers/fixturenet-eth-plugeth-arm-test

@@ -0,0 +1,2 @@
+Change this file to trigger running the fixturenet-eth-plugeth-arm-test CI job
+

.gitea/workflows/triggers/test-container-registry

@@ -0,0 +1 @@
+Change this file to trigger running the test-container-registry CI job

.gitea/workflows/triggers/test-database

@@ -0,0 +1,2 @@
+Change this file to trigger running the test-database CI job
+Trigger test run

README.md

@@ -29,10 +29,10 @@ chmod +x ~/.docker/cli-plugins/docker-compose
 Next decide on a directory where you would like to put the stack-orchestrator program. Typically this would be 
 a "user" binary directory such as `~/bin` or perhaps `/usr/local/laconic` or possibly just the current working directory.
 
-Now, having selected that directory, download the latest release from [this page](https://github.com/cerc-io/stack-orchestrator/tags) into it (we're using `~/bin` below for concreteness but edit to suit if you selected a different directory). Also be sure that the destination directory exists and is writable:
+Now, having selected that directory, download the latest release from [this page](https://git.vdb.to/cerc-io/stack-orchestrator/tags) into it (we're using `~/bin` below for concreteness but edit to suit if you selected a different directory). Also be sure that the destination directory exists and is writable:
 
 ```bash
-curl -L -o ~/bin/laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+curl -L -o ~/bin/laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
 ```
 
 Give it execute permissions:
@@ -52,7 +52,7 @@ Version: 1.1.0-7a607c2-202304260513
 Save the distribution url to `~/.laconic-so/config.yml`:
 ```bash
 mkdir ~/.laconic-so
-echo "distribution-url: https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so" >  ~/.laconic-so/config.yml
+echo "distribution-url: https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so" >  ~/.laconic-so/config.yml
 ```
 
 ### Update

docs/CONTRIBUTING.md

@@ -26,7 +26,7 @@ In addition to the pre-requisites listed in the [README](/README.md), the follow
 
 1. Clone this repository:
    ```
-   $ git clone https://github.com/cerc-io/stack-orchestrator.git
+   $ git clone https://git.vdb.to/cerc-io/stack-orchestrator.git
    ```
 
 2. Enter the project directory:

docs/adding-a-new-stack.md

@@ -1,10 +1,10 @@
 # Adding a new stack
 
-See [this PR](https://github.com/cerc-io/stack-orchestrator/pull/434) for an example of how to currently add a minimal stack to stack orchestrator. The [reth stack](https://github.com/cerc-io/stack-orchestrator/pull/435) is another good example.
+See [this PR](https://git.vdb.to/cerc-io/stack-orchestrator/pull/434) for an example of how to currently add a minimal stack to stack orchestrator. The [reth stack](https://git.vdb.to/cerc-io/stack-orchestrator/pull/435) is another good example.
 
 For external developers, we recommend forking this repo and adding your stack directly to your fork. This initially requires running in "developer mode" as described [here](/docs/CONTRIBUTING.md). Check out the [Namada stack](https://github.com/vknowable/stack-orchestrator/blob/main/app/data/stacks/public-namada/digitalocean_quickstart.md) from Knowable to see how that is done.
 
-Core to the feature completeness of stack orchestrator is to [decouple the tool functionality from payload](https://github.com/cerc-io/stack-orchestrator/issues/315) which will no longer require forking to add a stack.
+Core to the feature completeness of stack orchestrator is to [decouple the tool functionality from payload](https://git.vdb.to/cerc-io/stack-orchestrator/issues/315) which will no longer require forking to add a stack.
 
 ## Example
 

docs/fetching-containers.md

@@ -0,0 +1,9 @@
+# Fetching pre-built container images
+When Stack Orchestrator deploys a stack containing a suite of one or more containers it expects images for those containers to be on the local machine with a tag of the form `<image-name>:local` Images for these containers can be built from source (and optionally base container images from public registries) with the `build-containers` subcommand. 
+
+However, the task of building a large number of containers from source may consume considerable time and machine resources. This is where the `fetch-containers` subcommand steps in. It is designed to work exactly like `build-containers` but instead the images, pre-built, are fetched from an image registry then re-tagged for deployment. It can be used in place of `build-containers` for any stack provided the necessary containers, built for the local machine architecture (e.g. arm64 or x86-64) have already been published in an image registry.
+## Usage
+To use `fetch-containers`, provide an image registry path, a username and token/password with read access to the registry, and optionally specify `--force-local-overwrite`. If this argument is not specified, if there is already a locally built or previously fetched image for a stack container on the machine, it will not be overwritten and a warning issued.
+```
+$ laconic-so --stack mobymask-v3-demo fetch-containers --image-registry git.vdb.to/cerc-io --registry-username <registry-user> --registry-token <registry-token> --force-local-overwrite
+```

docs/laconicd-with-console.md

@@ -15,132 +15,139 @@ To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the
 16 GB Memory / 8 Intel vCPUs / 160 GB Disk.
 
 1. Login to the droplet as root (either by SSH key or password set in the DO console)
+    ```
+    ssh root@IP
+    ```
 
-```
-ssh root@IP
-```
+1. Get the install script, give it executable permissions, and run it:
 
-2. Get the install script, give it executable permissions, and run it:
+    ```
+    curl -o install.sh https://raw.githubusercontent.com/cerc-io/stack-orchestrator/main/scripts/quick-install-linux.sh
+    ```
+    ```
+    chmod +x install.sh
+    ```
+    ```
+    bash install.sh
+    ```
 
-```
-curl -o install.sh https://raw.githubusercontent.com/cerc-io/stack-orchestrator/main/scripts/quick-install-linux.sh
-```
-```
-chmod +x install.sh
-```
-```
-bash install.sh
-```
+1. Confirm docker was installed and activate the changes in `~/.profile`:
 
-3. Confirm docker was installed and activate the changes in `~/.profile`:
+    ```
+    docker run hello-world
+   ```
+   ```
+   source ~/.profile
+   ```
 
-```
-docker run hello-world
-```
-```
-source ~/.profile
-```
+1. Verify installation:
 
-4. Verify installation:
-
-```
-laconic-so version
-```
+    ```
+    laconic-so version
+    ```
 
 ## Setup the laconic fixturenet stack
 
 1. Get the repositories
 
-```
-laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
-```
+    ```
+    laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
+    ```
 
-2. Set this environment variable to the Laconic self-hosted Gitea instance:
+1. Build the containers:
 
-```
-export CERC_NPM_REGISTRY_URL=https://git.vdb.to/api/packages/cerc-io/npm/
-```
+    ```
+    laconic-so --stack fixturenet-laconic-loaded build-containers
+    ```
 
-3. Build the containers:
+    It's possible to run into an `ESOCKETTIMEDOUT` error, e.g., `error An unexpected error occurred: "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.11.3.tgz: ESOCKETTIMEDOUT"`. This may happen even if you have a great internet connection. In that case, re-run the `build-containers` command.
 
-```
-laconic-so --stack fixturenet-laconic-loaded build-containers
-```
 
-It's possible to run into an `ESOCKETTIMEDOUT` error, e.g., `error An unexpected error occurred: "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.11.3.tgz: ESOCKETTIMEDOUT"`. This may happen even if you have a great internet connection. In that case, re-run the `build-containers` command.
+1. Set this environment variable to your droplet's IP address or fully qualified DNS host name if it has one:
 
-4. Set this environment variable to your droplet's IP address:
+    ```
+    export BACKEND_ENDPOINT=http://<your-IP-or-hostname>:9473
+    ```
+    e.g.
+    ```
+    export BACKEND_ENDPOINT=http://my-test-server.example.com:9473
+    ```
 
-```
-export LACONIC_HOSTED_ENDPOINT=http://<your-IP>
-```
+1. Create a deployment directory for the stack:
+    ```
+    laconic-so --stack fixturenet-laconic-loaded deploy init --output laconic-loaded.spec --map-ports-to-host any-same --config LACONIC_HOSTED_ENDPOINT=$BACKEND_ENDPOINT
+    ```
+    ```
+    laconic-so --stack fixturenet-laconic-loaded deploy create --deployment-dir laconic-loaded-deployment --spec-file laconic-loaded.spec
+    ```
+2. Start the stack:
 
-5. Deploy the stack:
+    ```
+    laconic-so deployment --dir laconic-loaded-deployment start
+    ```
 
-```
-laconic-so --stack fixturenet-laconic-loaded deploy up
-```
+3. Check the logs:
 
-6. Check the logs:
+    ```
+    laconic-so deployment --dir laconic-loaded-deployment logs
+    ```
 
-```
-laconic-so --stack fixturenet-laconic-loaded deploy logs
-```
+    You'll see output from `laconicd` and the block height should be >1 to confirm it is running:
 
-You'll see output from `laconicd` and the block height should be >1 to confirm it is running:
+    ```
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:29PM INF indexed block exents height=12 module=txindex server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF Timed out dur=4976.960115 height=13 module=consensus round=0 server=node step=1
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4","parts":{"hash":"22411A20B7F14CDA33244420FBDDAF24450C0628C7A06034FF22DAC3699DDCC8","total":1}},"height":13,"pol_round":-1,"round":0,"signature":"DEuqnaQmvyYbUwckttJmgKdpRu6eVm9i+9rQ1pIrV2PidkMNdWRZBLdmNghkIrUzGbW8Xd7UVJxtLRmwRASgBg==","timestamp":"2023-04-18T21:30:01.49450663Z"} server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received complete proposal block hash=D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4 height=13 module=consensus server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF finalizing commit of block hash={} height=13 module=consensus num_txs=0 root=1A8CA1AF139CCC80EC007C6321D8A63A46A793386EE2EDF9A5CA0AB2C90728B7 server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF minted coins from module account amount=2059730459416582643aphoton from=mint module=x/bank
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF executed block height=13 module=state num_invalid_txs=0 num_valid_txs=0 server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF commit synced commit=436F6D6D697449447B5B363520313037203630203232372039352038352032303820313334203231392032303520313433203130372031343920313431203139203139322038362031323720362031383520323533203137362031333820313735203135392031383620323334203135382031323120313431203230342037335D3A447D
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF committed state app_hash=416B3CE35F55D086DBCD8F6B958D13C0567F06B9FDB08AAF9FBAEA9E798DCC49 height=13 module=state num_txs=0 server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF indexed block exents height=13 module=txindex server=node
+    ```
 
-```
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:29PM INF indexed block exents height=12 module=txindex server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF Timed out dur=4976.960115 height=13 module=consensus round=0 server=node step=1
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4","parts":{"hash":"22411A20B7F14CDA33244420FBDDAF24450C0628C7A06034FF22DAC3699DDCC8","total":1}},"height":13,"pol_round":-1,"round":0,"signature":"DEuqnaQmvyYbUwckttJmgKdpRu6eVm9i+9rQ1pIrV2PidkMNdWRZBLdmNghkIrUzGbW8Xd7UVJxtLRmwRASgBg==","timestamp":"2023-04-18T21:30:01.49450663Z"} server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received complete proposal block hash=D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4 height=13 module=consensus server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF finalizing commit of block hash={} height=13 module=consensus num_txs=0 root=1A8CA1AF139CCC80EC007C6321D8A63A46A793386EE2EDF9A5CA0AB2C90728B7 server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF minted coins from module account amount=2059730459416582643aphoton from=mint module=x/bank
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF executed block height=13 module=state num_invalid_txs=0 num_valid_txs=0 server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF commit synced commit=436F6D6D697449447B5B363520313037203630203232372039352038352032303820313334203231392032303520313433203130372031343920313431203139203139322038362031323720362031383520323533203137362031333820313735203135392031383620323334203135382031323120313431203230342037335D3A447D
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF committed state app_hash=416B3CE35F55D086DBCD8F6B958D13C0567F06B9FDB08AAF9FBAEA9E798DCC49 height=13 module=state num_txs=0 server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF indexed block exents height=13 module=txindex server=node
-```
+4. Confirm operation of the registry CLI:
 
-7. Confirm operation of the registry CLI:
+   ```
+   laconic-so deployment --dir laconic-loaded-deployment exec cli "laconic cns status"
+   ```
 
-```
-laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns status"
-```
-
-```
-{
-  "version": "0.3.0",
-  "node": {
-    "id": "4216af2ac9f68bda33a38803fc1b5c9559312c1d",
-    "network": "laconic_9000-1",
-    "moniker": "localtestnet"
-  },
-  "sync": {
-    "latest_block_hash": "1BDF4CB9AE2390DA65BCF997C83133C18014FCDDCAE03708488F0B56FCEEA429",
-    "latest_block_height": "5",
-    "latest_block_time": "2023-08-09 16:00:30.386903172 +0000 UTC",
-    "catching_up": false
-  },
-  "validator": {
-    "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
-    "voting_power": "1000000000000000"
-  },
-  "validators": [
-    {
-      "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
-      "voting_power": "1000000000000000",
-      "proposer_priority": "0"
-    }
-  ],
-  "num_peers": "0",
-  "peers": [],
-  "disk_usage": "292.0K"
-}
-```
+   ```
+   {
+     "version": "0.3.0",
+     "node": {
+       "id": "4216af2ac9f68bda33a38803fc1b5c9559312c1d",
+       "network": "laconic_9000-1",
+       "moniker": "localtestnet"
+     },
+     "sync": {
+       "latest_block_hash": "1BDF4CB9AE2390DA65BCF997C83133C18014FCDDCAE03708488F0B56FCEEA429",
+       "latest_block_height": "5",
+       "latest_block_time": "2023-08-09 16:00:30.386903172 +0000 UTC",
+       "catching_up": false
+     },
+     "validator": {
+       "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
+       "voting_power": "1000000000000000"
+     },
+     "validators": [
+       {
+         "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
+         "voting_power": "1000000000000000",
+         "proposer_priority": "0"
+       }
+     ],
+     "num_peers": "0",
+     "peers": [],
+     "disk_usage": "292.0K"
+   }
+   ```
 
 ## Configure Digital Ocean firewall
 
+(Note this step may not be necessary depending on the droplet image used)
+
 Let's open some ports.
 
 1. In the Digital Ocean web console, navigate to your droplet's main page. Select the "Networking" tab and scroll down to "Firewall".

docs/spec.md

@@ -1,6 +1,6 @@
 # Specification
 
-Note: this page is out of date (but still useful) - it will no longer be useful once stacks are [decoupled from the tool functionality](https://github.com/cerc-io/stack-orchestrator/issues/315).
+Note: this page is out of date (but still useful) - it will no longer be useful once stacks are [decoupled from the tool functionality](https://git.vdb.to/cerc-io/stack-orchestrator/issues/315).
 
 ## Implementation
 

requirements.txt

@@ -10,3 +10,4 @@ pydantic==1.10.9
 tomli==2.0.1
 validators==0.22.0
 kubernetes>=28.1.0
+humanfriendly>=10.0

scripts/cloud-init-dev-mode-install.yaml

@@ -41,4 +41,4 @@ runcmd:
   - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
   - systemctl enable docker
   - systemctl start docker
-  - git clone https://github.com/cerc-io/stack-orchestrator.git /home/ubuntu/stack-orchestrator
+  - git clone https://git.vdb.to/cerc-io/stack-orchestrator.git /home/ubuntu/stack-orchestrator

scripts/cloud-init-user-mode-install.yaml

@@ -31,5 +31,5 @@ runcmd:
   - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
   - systemctl enable docker
   - systemctl start docker
-  - curl -L -o /usr/local/bin/laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+  - curl -L -o /usr/local/bin/laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
   - chmod +x /usr/local/bin/laconic-so

scripts/quick-deploy-test.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Beginnings of a script to quickly spin up and test a deployment
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+if [[ -n "$1" ]]; then
+	stack_name=$1
+else
+	stack_name="test"
+fi
+spec_file_name="${stack_name}-spec.yml"
+deployment_dir_name="${stack_name}-deployment"
+rm -f ${spec_file_name}
+rm -rf ${deployment_dir_name}
+laconic-so --stack ${stack_name} deploy --deploy-to compose init --output ${spec_file_name}
+laconic-so --stack ${stack_name} deploy --deploy-to compose create --deployment-dir ${deployment_dir_name} --spec-file ${spec_file_name}
+#laconic-so deployment --dir ${deployment_dir_name} start
+#laconic-so deployment --dir ${deployment_dir_name} ps
+#laconic-so deployment --dir ${deployment_dir_name} stop

scripts/quick-install-linux.sh

@@ -137,7 +137,7 @@ fi
 echo "**************************************************************************************"
 echo "Installing laconic-so"
 # install latest `laconic-so`
-distribution_url=https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+distribution_url=https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
 install_filename=${install_dir}/laconic-so
 mkdir -p  ${install_dir}
 curl -L -o ${install_filename} ${distribution_url}

setup.py

@@ -13,7 +13,7 @@ setup(
     description='Orchestrates deployment of the Laconic stack',
     long_description=long_description,
     long_description_content_type="text/markdown",
-    url='https://github.com/cerc-io/stack-orchestrator',
+    url='https://git.vdb.to/cerc-io/stack-orchestrator',
     py_modules=['stack_orchestrator'],
     packages=find_packages(),
     install_requires=[requirements],

stack_orchestrator/build/build_containers.py

@@ -25,10 +25,13 @@ import sys
 from decouple import config
 import subprocess
 import click
-import importlib.resources
 from pathlib import Path
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
 from stack_orchestrator.base import get_npm_registry_url
+from stack_orchestrator.build.build_types import BuildContext
+from stack_orchestrator.build.publish import publish_image
+from stack_orchestrator.build.build_util import get_containers_in_scope
 
 # TODO: find a place for this
 #    epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
@@ -59,69 +62,58 @@ def make_container_build_env(dev_root_path: str,
     return container_build_env
 
 
-def process_container(stack: str,
-                      container,
-                      container_build_dir: str,
-                      container_build_env: dict,
-                      dev_root_path: str,
-                      quiet: bool,
-                      verbose: bool,
-                      dry_run: bool,
-                      continue_on_error: bool,
-                      ):
-    if not quiet:
-        print(f"Building: {container}")
+def process_container(build_context: BuildContext) -> bool:
+    if not opts.o.quiet:
+        print(f"Building: {build_context.container}")
 
-    default_container_tag = f"{container}:local"
-    container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
+    default_container_tag = f"{build_context.container}:local"
+    build_context.container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
 
     # Check if this is in an external stack
-    if stack_is_external(stack):
-        container_parent_dir = Path(stack).joinpath("container-build")
-        temp_build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
+    if stack_is_external(build_context.stack):
+        container_parent_dir = Path(build_context.stack).joinpath("container-build")
+        temp_build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
         temp_build_script_filename = temp_build_dir.joinpath("build.sh")
         # Now check if the container exists in the external stack.
         if not temp_build_script_filename.exists():
             # If not, revert to building an internal container
-            container_parent_dir = container_build_dir
+            container_parent_dir = build_context.container_build_dir
     else:
-        container_parent_dir = container_build_dir
+        container_parent_dir = build_context.container_build_dir
 
-    build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
+    build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
     build_script_filename = build_dir.joinpath("build.sh")
 
-    if verbose:
+    if opts.o.verbose:
         print(f"Build script filename: {build_script_filename}")
     if os.path.exists(build_script_filename):
         build_command = build_script_filename.as_posix()
     else:
-        if verbose:
+        if opts.o.verbose:
             print(f"No script file found: {build_script_filename}, using default build script")
-        repo_dir = container.split('/')[1]
+        repo_dir = build_context.container.split('/')[1]
         # TODO: make this less of a hack -- should be specified in some metadata somewhere
         # Check if we have a repo for this container. If not, set the context dir to the container-build subdir
-        repo_full_path = os.path.join(dev_root_path, repo_dir)
+        repo_full_path = os.path.join(build_context.dev_root_path, repo_dir)
         repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
-        build_command = os.path.join(container_build_dir,
+        build_command = os.path.join(build_context.container_build_dir,
                                      "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
-    if not dry_run:
+    if not opts.o.dry_run:
         # No PATH at all causes failures with podman.
-        if "PATH" not in container_build_env:
-            container_build_env["PATH"] = os.environ["PATH"]
-        if verbose:
-            print(f"Executing: {build_command} with environment: {container_build_env}")
-        build_result = subprocess.run(build_command, shell=True, env=container_build_env)
-        if verbose:
+        if "PATH" not in build_context.container_build_env:
+            build_context.container_build_env["PATH"] = os.environ["PATH"]
+        if opts.o.verbose:
+            print(f"Executing: {build_command} with environment: {build_context.container_build_env}")
+        build_result = subprocess.run(build_command, shell=True, env=build_context.container_build_env)
+        if opts.o.verbose:
             print(f"Return code is: {build_result.returncode}")
         if build_result.returncode != 0:
-            print(f"Error running build for {container}")
-            if not continue_on_error:
-                print("FATAL Error: container build failed and --continue-on-error not set, exiting")
-                sys.exit(1)
-            else:
-                print("****** Container Build Error, continuing because --continue-on-error is set")
+            return False
+        else:
+            return True
     else:
         print("Skipped")
+        return True
 
 
 @click.command()
@@ -129,17 +121,14 @@ def process_container(stack: str,
 @click.option('--exclude', help="don\'t build these containers")
 @click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
 @click.option("--extra-build-args", help="Supply extra arguments to build")
+@click.option("--publish-images", is_flag=True, default=False, help="Publish the built images in the specified image registry")
+@click.option("--image-registry", help="Specify the image registry for --publish-images")
 @click.pass_context
-def command(ctx, include, exclude, force_rebuild, extra_build_args):
+def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_images, image_registry):
     '''build the set of containers required for a complete stack'''
 
-    quiet = ctx.obj.quiet
-    verbose = ctx.obj.verbose
-    dry_run = ctx.obj.dry_run
-    debug = ctx.obj.debug
     local_stack = ctx.obj.local_stack
     stack = ctx.obj.stack
-    continue_on_error = ctx.obj.continue_on_error
 
     # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
     container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
@@ -150,39 +139,45 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
     else:
         dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
 
-    if not quiet:
+    if not opts.o.quiet:
         print(f'Dev Root is: {dev_root_path}')
 
     if not os.path.isdir(dev_root_path):
         print('Dev root directory doesn\'t exist, creating')
 
-    # See: https://stackoverflow.com/a/20885799/1701505
-    from stack_orchestrator import data
-    with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
-        all_containers = container_list_file.read().splitlines()
+    if publish_images:
+        if not image_registry:
+            error_exit("--image-registry must be supplied with --publish-images")
 
-    containers_in_scope = []
-    if stack:
-        stack_config = get_parsed_stack_config(stack)
-        containers_in_scope = stack_config['containers']
-    else:
-        containers_in_scope = all_containers
-
-    if verbose:
-        print(f'Containers: {containers_in_scope}')
-        if stack:
-            print(f"Stack: {stack}")
+    containers_in_scope = get_containers_in_scope(stack)
 
     container_build_env = make_container_build_env(dev_root_path,
                                                    container_build_dir,
-                                                   debug,
+                                                   opts.o.debug,
                                                    force_rebuild,
                                                    extra_build_args)
 
     for container in containers_in_scope:
         if include_exclude_check(container, include, exclude):
-            process_container(stack, container, container_build_dir, container_build_env,
-                              dev_root_path, quiet, verbose, dry_run, continue_on_error)
+
+            build_context = BuildContext(
+                stack,
+                container,
+                container_build_dir,
+                container_build_env,
+                dev_root_path
+            )
+            result = process_container(build_context)
+            if result:
+                if publish_images:
+                    publish_image(f"{container}:local", image_registry)
+            else:
+                print(f"Error running build for {build_context.container}")
+                if not opts.o.continue_on_error:
+                    error_exit("container build failed and --continue-on-error not set, exiting")
+                    sys.exit(1)
+                else:
+                    print("****** Container Build Error, continuing because --continue-on-error is set")
         else:
-            if verbose:
+            if opts.o.verbose:
                 print(f"Excluding: {container}")

stack_orchestrator/build/build_types.py

@@ -0,0 +1,29 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Mapping
+
+
+@dataclass
+class BuildContext:
+    stack: str
+    container: str
+    container_build_dir: Path
+    container_build_env: Mapping[str,str]
+    dev_root_path: str
+

stack_orchestrator/build/build_util.py

@@ -0,0 +1,43 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import importlib.resources
+
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import get_parsed_stack_config, warn_exit
+
+
+def get_containers_in_scope(stack: str):
+
+    # See: https://stackoverflow.com/a/20885799/1701505
+    from stack_orchestrator import data
+    with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
+        all_containers = container_list_file.read().splitlines()
+
+    containers_in_scope = []
+    if stack:
+        stack_config = get_parsed_stack_config(stack)
+        if "containers" not in stack_config or stack_config["containers"] is None:
+            warn_exit(f"stack {stack} does not define any containers")
+        containers_in_scope = stack_config['containers']
+    else:
+        containers_in_scope = all_containers
+
+    if opts.o.verbose:
+        print(f'Containers: {containers_in_scope}')
+        if stack:
+            print(f"Stack: {stack}")
+
+    return containers_in_scope

stack_orchestrator/build/build_webapp.py

@@ -21,11 +21,14 @@
 # TODO: display the available list of containers; allow re-build of either all or specific containers
 
 import os
+import sys
+
 from decouple import config
 import click
 from pathlib import Path
 from stack_orchestrator.build import build_containers
-from stack_orchestrator.deploy.webapp.util import determine_base_container
+from stack_orchestrator.deploy.webapp.util import determine_base_container, TimedLogger
+from stack_orchestrator.build.build_types import BuildContext
 
 
 @click.command()
@@ -37,26 +40,25 @@ from stack_orchestrator.deploy.webapp.util import determine_base_container
 @click.pass_context
 def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag):
     '''build the specified webapp container'''
+    logger = TimedLogger()
 
     quiet = ctx.obj.quiet
-    verbose = ctx.obj.verbose
-    dry_run = ctx.obj.dry_run
     debug = ctx.obj.debug
+    verbose = ctx.obj.verbose
     local_stack = ctx.obj.local_stack
     stack = ctx.obj.stack
-    continue_on_error = ctx.obj.continue_on_error
 
     # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
     container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
 
     if local_stack:
         dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
-        print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
+        logger.log(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
     else:
         dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
 
-    if not quiet:
-        print(f'Dev Root is: {dev_root_path}')
+    if verbose:
+        logger.log(f'Dev Root is: {dev_root_path}')
 
     if not base_container:
         base_container = determine_base_container(source_repo)
@@ -65,8 +67,23 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
                                                                     force_rebuild, extra_build_args)
 
-    build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
-                                       verbose, dry_run, continue_on_error)
+    if verbose:
+        logger.log(f"Building base container: {base_container}")
+
+    build_context_1 = BuildContext(
+        stack,
+        base_container,
+        container_build_dir,
+        container_build_env,
+        dev_root_path,
+    )
+    ok = build_containers.process_container(build_context_1)
+    if not ok:
+        logger.log("ERROR: Build failed.")
+        sys.exit(1)
+
+    if verbose:
+        logger.log(f"Base container {base_container} build finished.")
 
     # Now build the target webapp.  We use the same build script, but with a different Dockerfile and work dir.
     container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
@@ -76,9 +93,25 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
                                                                           "Dockerfile.webapp")
     if not tag:
         webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
-        container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
-    else:
-        container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag
+        tag = f"cerc/{webapp_name}:local"
 
-    build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
-                                       verbose, dry_run, continue_on_error)
+    container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag
+
+    if verbose:
+        logger.log(f"Building app container: {tag}")
+
+    build_context_2 = BuildContext(
+        stack,
+        base_container,
+        container_build_dir,
+        container_build_env,
+        dev_root_path,
+    )
+    ok = build_containers.process_container(build_context_2)
+    if not ok:
+        logger.log("ERROR: Build failed.")
+        sys.exit(1)
+
+    if verbose:
+        logger.log(f"App container {base_container} build finished.")
+        logger.log("build-webapp complete", show_step_time=False, show_total_time=True)

stack_orchestrator/build/fetch_containers.py

@@ -0,0 +1,195 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import click
+from dataclasses import dataclass
+import json
+import platform
+from python_on_whales import DockerClient
+from python_on_whales.components.manifest.cli_wrapper import ManifestCLI, ManifestList
+from python_on_whales.utils import run
+import requests
+from typing import List
+
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import include_exclude_check, error_exit
+from stack_orchestrator.build.build_util import get_containers_in_scope
+
+# Experimental fetch-container command
+
+
+@dataclass
+class RegistryInfo:
+    registry: str
+    registry_username: str
+    registry_token: str
+
+
+# Extending this code to support the --verbose option, cnosider contributing upstream
+# https://github.com/gabrieldemarmiesse/python-on-whales/blob/master/python_on_whales/components/manifest/cli_wrapper.py#L129
+class ExtendedManifestCLI(ManifestCLI):
+    def inspect_verbose(self, x: str) -> ManifestList:
+        """Returns a Docker manifest list object."""
+        json_str = run(self.docker_cmd + ["manifest", "inspect", "--verbose", x])
+        return json.loads(json_str)
+
+
+def _local_tag_for(container: str):
+    return f"{container}:local"
+
+
+# See: https://docker-docs.uclv.cu/registry/spec/api/
+# Emulate this:
+# $ curl -u "my-username:my-token" -X GET "https://<container-registry-hostname>/v2/cerc-io/cerc/test-container/tags/list"
+# {"name":"cerc-io/cerc/test-container","tags":["202402232130","202402232208"]}
+def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List[str]:
+    # registry looks like: git.vdb.to/cerc-io
+    registry_parts = registry_info.registry.split("/")
+    url = f"https://{registry_parts[0]}/v2/{registry_parts[1]}/{container}/tags/list"
+    if opts.o.debug:
+        print(f"Fetching tags from: {url}")
+    response = requests.get(url, auth=(registry_info.registry_username, registry_info.registry_token))
+    if response.status_code == 200:
+        tag_info = response.json()
+        if opts.o.debug:
+            print(f"container tags list: {tag_info}")
+        tags_array = tag_info["tags"]
+        return tags_array
+    else:
+        error_exit(f"failed to fetch tags from image registry, status code: {response.status_code}")
+
+
+def _find_latest(candidate_tags: List[str]):
+    # Lex sort should give us the latest first
+    sorted_candidates = sorted(candidate_tags)
+    if opts.o.debug:
+        print(f"sorted candidates: {sorted_candidates}")
+    return sorted_candidates[-1]
+
+
+def _filter_for_platform(container: str, 
+                         registry_info: RegistryInfo,
+                         tag_list: List[str]) -> List[str] :
+    filtered_tags = []
+    this_machine = platform.machine()
+    # Translate between Python and docker platform names
+    if this_machine == "x86_64":
+        this_machine = "amd64"
+    if this_machine == "aarch64":
+        this_machine = "arm64"
+    if opts.o.debug:
+        print(f"Python says the architecture is: {this_machine}")
+    docker = DockerClient()
+    for tag in tag_list:
+        remote_tag = f"{registry_info.registry}/{container}:{tag}"
+        manifest_cmd = ExtendedManifestCLI(docker.client_config)
+        manifest = manifest_cmd.inspect_verbose(remote_tag)
+        if opts.o.debug:
+            print(f"manifest: {manifest}")
+        image_architecture =  manifest["Descriptor"]["platform"]["architecture"]
+        if opts.o.debug:
+            print(f"image_architecture: {image_architecture}")
+        if this_machine == image_architecture:
+            filtered_tags.append(tag)
+    if opts.o.debug:
+        print(f"Tags filtered for platform: {filtered_tags}")
+    return filtered_tags
+
+
+def _get_latest_image(container: str, registry_info: RegistryInfo):
+    all_tags = _get_tags_for_container(container, registry_info)
+    tags_for_platform = _filter_for_platform(container, registry_info, all_tags)
+    if len(tags_for_platform) > 0:
+        latest_tag = _find_latest(tags_for_platform)
+        return f"{container}:{latest_tag}"
+    else:
+        return None
+
+
+def _fetch_image(tag: str, registry_info: RegistryInfo):
+    docker = DockerClient()
+    remote_tag = f"{registry_info.registry}/{tag}"
+    if opts.o.debug:
+        print(f"Attempting to pull this image: {remote_tag}")
+    docker.image.pull(remote_tag)
+
+
+def _exists_locally(container: str):
+    docker = DockerClient()
+    return docker.image.exists(_local_tag_for(container))
+
+
+def _add_local_tag(remote_tag: str, registry: str, local_tag: str):
+    docker = DockerClient()
+    docker.image.tag(f"{registry}/{remote_tag}", local_tag)
+
+
+@click.command()
+@click.option('--include', help="only fetch these containers")
+@click.option('--exclude', help="don\'t fetch these containers")
+@click.option("--force-local-overwrite", is_flag=True, default=False, help="Overwrite a locally built image, if present")
+@click.option("--image-registry", required=True, help="Specify the image registry to fetch from")
+@click.option("--registry-username", required=True, help="Specify the image registry username")
+@click.option("--registry-token", required=True, help="Specify the image registry access token")
+@click.pass_context
+def command(ctx, include, exclude, force_local_overwrite, image_registry, registry_username, registry_token):
+    '''EXPERIMENTAL: fetch the images for a stack from remote registry'''
+
+    registry_info = RegistryInfo(image_registry, registry_username, registry_token)
+    docker = DockerClient()
+    if not opts.o.quiet:
+        print("Logging into container registry:")
+    docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token)
+    # Generate list of target containers
+    stack = ctx.obj.stack
+    containers_in_scope = get_containers_in_scope(stack)
+    all_containers_found = True
+    for container in containers_in_scope:
+        local_tag = _local_tag_for(container)
+        if include_exclude_check(container, include, exclude):
+            if opts.o.debug:
+                print(f"Processing: {container}")
+            # For each container, attempt to find the latest of a set of
+            # images with the correct name and platform in the specified registry
+            image_to_fetch = _get_latest_image(container, registry_info)
+            if not image_to_fetch:
+                print(f"Warning: no image found to fetch for container: {container}")
+                all_containers_found = False
+                continue
+            if opts.o.debug:
+                print(f"Fetching: {image_to_fetch}")
+            _fetch_image(image_to_fetch, registry_info)
+            # Now check if the target container already exists exists locally already
+            if (_exists_locally(container)):
+                if not opts.o.quiet:
+                    print(f"Container image {container} already exists locally")
+                # if so, fail unless the user specified force-local-overwrite
+                if (force_local_overwrite):
+                    # In that case remove the existing :local tag
+                    if not opts.o.quiet:
+                        print(f"Warning: overwriting local tag from this image: {container} because "
+                              "--force-local-overwrite was specified")
+                else:
+                    if not opts.o.quiet:
+                        print(f"Skipping local tagging for this image: {container} because that would "
+                              "overwrite an existing :local tagged image, use --force-local-overwrite to do so.")
+                    continue
+            # Tag the fetched image with the :local tag
+            _add_local_tag(image_to_fetch, image_registry, local_tag)
+        else:
+            if opts.o.verbose:
+                print(f"Excluding: {container}")
+    if not all_containers_found:
+        print("Warning: couldn't find usable images for one or more containers, this stack will not deploy")

stack_orchestrator/build/publish.py

@@ -0,0 +1,48 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from datetime import datetime
+from python_on_whales import DockerClient
+
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import error_exit
+
+
+def _publish_tag_for_image(local_image_tag: str, remote_repo: str, version: str):
+    # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
+    (image_name, image_version) = local_image_tag.split(":")
+    if image_version == "local":
+        return f"{remote_repo}/{image_name}:{version}"
+    else:
+        error_exit("Asked to publish a non-locally built image")
+
+
+def publish_image(local_tag, registry):
+    if opts.o.verbose:
+        print(f"Publishing this image: {local_tag} to this registry: {registry}")
+    docker = DockerClient()
+    # Figure out the target image tag
+    # Eventually this version will be generated from the source repo state
+    # Using a timestemp is an intermediate step
+    version = datetime.now().strftime("%Y%m%d%H%M")
+    remote_tag = _publish_tag_for_image(local_tag, registry, version)
+    # Tag the image thus
+    if opts.o.debug:
+        print(f"Tagging {local_tag} to {remote_tag}")
+    docker.image.tag(local_tag, remote_tag)
+    # Push it to the desired registry
+    if opts.o.verbose:
+        print(f"Pushing image {remote_tag}")
+    docker.image.push(remote_tag)

stack_orchestrator/constants.py

@@ -27,6 +27,12 @@ kube_config_key = "kube-config"
 deploy_to_key = "deploy-to"
 network_key = "network"
 http_proxy_key = "http-proxy"
-image_resigtry_key = "image-registry"
+image_registry_key = "image-registry"
+configmaps_key = "configmaps"
+resources_key = "resources"
+volumes_key = "volumes"
+security_key = "security"
+annotations_key = "annotations"
+labels_key = "labels"
 kind_config_filename = "kind-config.yml"
 kube_config_filename = "kubeconfig.yml"

stack_orchestrator/data/compose/docker-compose-container-registry.yml

@@ -0,0 +1,15 @@
+services:
+  registry:
+    image: registry:2.8
+    restart: always
+    environment:
+      REGISTRY_LOG_LEVEL: ${REGISTRY_LOG_LEVEL}
+    volumes:
+      - config:/config:ro
+      - registry-data:/var/lib/registry
+    ports:
+      - "5000"
+
+volumes:
+  config:
+  registry-data:

stack_orchestrator/data/compose/docker-compose-mars-v2.yml

@@ -0,0 +1,12 @@
+version: "3.2"
+
+services:
+  mars:
+    image: cerc/mars-v2:local
+    restart: always
+    ports:
+      - "3000:3000"
+    environment:
+      - URL_OSMOSIS_REST=https://lcd-osmosis.blockapsis.com
+      - URL_OSMOSIS_RPC=https://rpc-osmosis.blockapsis.com
+      - WALLET_CONNECT_ID=0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x

stack_orchestrator/data/compose/docker-compose-mars.yml

@@ -0,0 +1,20 @@
+version: "3.2"
+
+services:
+  mars:
+    image: cerc/mars:local
+    restart: always
+    ports:
+      - "3000:3000"
+    environment:
+      - URL_OSMOSIS_GQL=https://osmosis-node.marsprotocol.io/GGSFGSFGFG34/osmosis-hive-front/graphql
+      - URL_OSMOSIS_REST=https://lcd-osmosis.blockapsis.com
+      - URL_OSMOSIS_RPC=https://rpc-osmosis.blockapsis.com
+      - URL_NEUTRON_GQL=https://neutron.rpc.p2p.world/qgrnU6PsQZA8F9S5Fb8Fn3tV3kXmMBl2M9bcc9jWLjQy8p/hive/graphql
+      - URL_NEUTRON_REST=https://rest-kralum.neutron-1.neutron.org
+      - URL_NEUTRON_RPC=https://rpc-kralum.neutron-1.neutron.org
+      - URL_NEUTRON_TEST_GQL=https://testnet-neutron-gql.marsprotocol.io/graphql
+      - URL_NEUTRON_TEST_REST=https://rest-palvus.pion-1.ntrn.tech
+      - URL_NEUTRON_TEST_RPC=https://rpc-palvus.pion-1.ntrn.tech
+      - WALLET_CONNECT_ID=0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x
+

stack_orchestrator/data/compose/docker-compose-ping-pub.yml

@@ -0,0 +1,8 @@
+version: "3.2"
+
+services:
+  ping-pub:
+    image: cerc/ping-pub:local
+    restart: always
+    ports:
+      - "5173:5173"

stack_orchestrator/data/compose/docker-compose-snowballtools-base-backend.yml

@@ -0,0 +1,13 @@
+services:
+  snowballtools-base-backend:
+    image: cerc/snowballtools-base-backend:local
+    restart: always
+    volumes:
+      - data:/data
+      - config:/config:ro
+    ports:
+      - 8000
+
+volumes:
+  data:
+  config:

stack_orchestrator/data/compose/docker-compose-test-database.yml

@@ -0,0 +1,20 @@
+services:
+
+  database:
+    image: cerc/test-database-container:local
+    restart: always
+    volumes:
+      - db-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: "test-user"
+      POSTGRES_DB: "test-db"
+      POSTGRES_PASSWORD: "password"
+      POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C"
+    ports:
+      - "5432"
+    
+  test-client:
+    image: cerc/test-database-client:local
+
+volumes:
+  db-data:

stack_orchestrator/data/compose/docker-compose-test.yml

@@ -5,12 +5,16 @@ services:
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_TEST_PARAM_1: ${CERC_TEST_PARAM_1:-FAILED}
+      CERC_TEST_PARAM_2: "CERC_TEST_PARAM_2_VALUE"
+      CERC_TEST_PARAM_3: ${CERC_TEST_PARAM_3:-FAILED}
     volumes:
-      - test-data:/data
+      - test-data-bind:/data
+      - test-data-auto:/data2
       - test-config:/config:ro
     ports:
       - "80"
 
 volumes:
-  test-data:
+  test-data-bind:
+  test-data-auto:
   test-config:

stack_orchestrator/data/compose/docker-compose-webapp-template.yml

@@ -5,4 +5,4 @@ services:
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
     ports:
-      - "3000"
+      - "80"

stack_orchestrator/data/container-build/cerc-mars-v2/build.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build the mars-v2 image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+docker build -t cerc/mars-v2:local -f ${CERC_REPO_BASE_DIR}/mars-v2-frontend/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/mars-v2-frontend

stack_orchestrator/data/container-build/cerc-mars/build.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build the mars image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+docker build -t cerc/mars:local -f ${CERC_REPO_BASE_DIR}/mars-interface/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/mars-interface

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -26,6 +26,8 @@ RUN \
     && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
     # Install semver
     && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
+    # Install pnpm
+    && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
     && npm cache clean --force > /dev/null 2>&1
 
 # [Optional] Uncomment this section to install additional OS packages.
@@ -35,6 +37,9 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"
 
+# We do this to get a yq binary from the published container, for the correct architecture we're building here
+COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
+
 # Expose port for http
 EXPOSE 80
 

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh

@@ -10,10 +10,12 @@ TRG_DIR="${3:-.next-r}"
 
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
-  if [ -f "yarn.lock" ]; then
-    CERC_BUILD_TOOL=npm
-  else
+  if [ -f "pnpm-lock.yaml" ]; then
+    CERC_BUILD_TOOL=pnpm
+  elif [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
+  else
+    CERC_BUILD_TOOL=npm
   fi
 fi
 
@@ -33,8 +35,8 @@ if [ -f ".env" ]; then
   rm -f $TMP_ENV
 fi
 
-for f in $(find "$TRG_DIR" -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do
-  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},();]' '\n' | egrep -o '^"CERC_RUNTIME_ENV_[^\"]+"'); do
+for f in $(find . -type f \( -regex '.*.html?' -or -regex ".*.[tj]s\(x\|on\)?$" \) | grep -v 'node_modules' | grep -v '.git'); do
+  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '["/\\{},();]' '\n' | tr -s "[']" '\n' | egrep -o -e '^CERC_RUNTIME_ENV_.+$' -e '^LACONIC_HOSTED_CONFIG_.+$'); do
     orig_name=$(echo -n "${e}" | sed 's/"//g')
     cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
     cur_val=$(echo -n "\$${cur_name}" | envsubst)

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -9,7 +9,9 @@ CERC_MIN_NEXTVER=13.4.2
 CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}"
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
-  if [ -f "yarn.lock" ]; then
+  if [ -f "pnpm-lock.yaml" ]; then
+    CERC_BUILD_TOOL=pnpm
+  elif [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
   else
     CERC_BUILD_TOOL=npm
@@ -113,7 +115,7 @@ if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VE
   mv package.json.$$ package.json
 fi
 
-$CERC_BUILD_TOOL install || exit 1
+time $CERC_BUILD_TOOL install || exit 1
 
 CUR_NEXT_VERSION=`jq -r '.version' node_modules/next/package.json`
 
@@ -136,9 +138,9 @@ to use for the build with:
 EOF
   cat package.json | jq ".dependencies.next = \"^$CERC_MIN_NEXTVER\"" > package.json.$$
   mv package.json.$$ package.json
-  $CERC_BUILD_TOOL install || exit 1
+  time $CERC_BUILD_TOOL install || exit 1
 fi
 
-$CERC_BUILD_TOOL run cerc_compile || exit 1
+time $CERC_BUILD_TOOL run cerc_compile || exit 1
 
 exit 0

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -16,7 +16,9 @@ trap ctrl_c INT
 
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
-  if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then
+  if [ -f "pnpm-lock.yaml" ]; then
+    CERC_BUILD_TOOL=pnpm
+  elif [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
   else
     CERC_BUILD_TOOL=npm

stack_orchestrator/data/container-build/cerc-ping-pub/build.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Build the ping pub image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+docker build -t cerc/ping-pub:local ${build_command_args} -f $CERC_REPO_BASE_DIR/explorer/Dockerfile $CERC_REPO_BASE_DIR/explorer

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/Dockerfile

@@ -0,0 +1,6 @@
+FROM cerc/snowballtools-base-backend-base:local
+
+WORKDIR /app/packages/backend
+COPY run.sh .
+
+ENTRYPOINT ["./run.sh"]

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/Dockerfile-base

@@ -0,0 +1,26 @@
+FROM ubuntu:22.04 as builder
+
+RUN apt update && \
+      apt install -y --no-install-recommends --no-install-suggests \
+      ca-certificates curl gnupg
+
+# Node
+ARG NODE_MAJOR=20
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
+      echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
+      apt update && apt install -y nodejs
+
+# npm setup
+RUN npm config set @cerc-io:registry https://git.vdb.to/api/packages/cerc-io/npm/ && npm install -g yarn
+
+COPY . /app/
+WORKDIR /app/
+
+RUN find . -name 'node_modules' | xargs -n1 rm -rf
+RUN yarn && yarn build --ignore frontend
+
+FROM cerc/webapp-base:local
+
+COPY --from=builder /app /app
+
+WORKDIR /app/packages/backend

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/build.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# Build cerc/webapp-deployer-backend
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/snowballtools-base-backend-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile-base ${CERC_REPO_BASE_DIR}/snowballtools-base
+docker build -t cerc/snowballtools-base-backend:local ${build_command_args} ${SCRIPT_DIR} 

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/run.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+
+LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
+if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
+  if [ -f "/config/laconic-hosted-config.yml" ]; then
+    LACONIC_HOSTED_CONFIG_FILE="/config/laconic-hosted-config.yml"
+  elif [ -f "/config/config.yml" ]; then
+    LACONIC_HOSTED_CONFIG_FILE="/config/config.yml"
+  fi
+fi
+
+if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
+  /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "`pwd`/dist"
+fi
+
+/scripts/apply-runtime-env.sh "`pwd`/dist"
+
+yarn start

stack_orchestrator/data/container-build/cerc-test-container/run.sh

@@ -1,22 +1,53 @@
 #!/usr/bin/env bash
 set -e
+
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
-# Test if the container's filesystem is old (run previously) or new
-EXISTSFILENAME=/data/exists
+
 echo "Test container starting"
-if [[ -f "$EXISTSFILENAME" ]];
-then
-    TIMESTAMP=`cat $EXISTSFILENAME`
-    echo "Filesystem is old, created: $TIMESTAMP" 
+
+DATA_DEVICE=$(df | grep "/data$" | awk '{ print $1 }')
+if [[ -n "$DATA_DEVICE" ]]; then
+  echo "/data: MOUNTED dev=${DATA_DEVICE}"
 else
-    echo "Filesystem is fresh"
-    echo `date` > $EXISTSFILENAME
+  echo "/data: not mounted"
 fi
+
+DATA2_DEVICE=$(df | grep "/data2$" | awk '{ print $1 }')
+if [[ -n "$DATA_DEVICE" ]]; then
+  echo "/data2: MOUNTED dev=${DATA2_DEVICE}"
+else
+  echo "/data2: not mounted"
+fi
+
+# Test if the container's filesystem is old (run previously) or new
+for d in /data /data2; do
+  if [[ -f "$d/exists" ]];
+  then
+      TIMESTAMP=`cat $d/exists`
+      echo "$d filesystem is old, created: $TIMESTAMP"
+  else
+      echo "$d filesystem is fresh"
+      echo `date` > $d/exists
+  fi
+done
+
 if [ -n "$CERC_TEST_PARAM_1" ]; then
   echo "Test-param-1: ${CERC_TEST_PARAM_1}"
 fi
+if [ -n "$CERC_TEST_PARAM_2" ]; then
+  echo "Test-param-2: ${CERC_TEST_PARAM_2}"
+fi
+if [ -n "$CERC_TEST_PARAM_3" ]; then
+  echo "Test-param-3: ${CERC_TEST_PARAM_3}"
+fi
+if [ -n "$CERC_TEST_PARAM_4" ]; then
+  echo "Test-param-4: ${CERC_TEST_PARAM_4}"
+fi
+if [ -n "$CERC_TEST_PARAM_5" ]; then
+  echo "Test-param-5: ${CERC_TEST_PARAM_5}"
+fi
 
 if [ -d "/config" ]; then
   echo "/config: EXISTS"

stack_orchestrator/data/container-build/cerc-test-database-client/Dockerfile

@@ -0,0 +1,12 @@
+FROM ubuntu:latest
+
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive && export DEBCONF_NOWARNINGS="yes" && \
+    apt-get install -y software-properties-common && \
+    apt-get install -y postgresql-client && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+EXPOSE 80
+
+COPY run.sh /app/run.sh
+
+ENTRYPOINT ["/app/run.sh"]

stack_orchestrator/data/container-build/cerc-test-database-client/build.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Build cerc/test-container
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+docker build -t cerc/test-database-client:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR

stack_orchestrator/data/container-build/cerc-test-database-client/run.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# TODO derive this from config
+database_url="postgresql://test-user:password@localhost:5432/test-db"
+psql_command="psql ${database_url}"
+program_name="Database test client:"
+
+wait_for_database_up () {
+    for i in {1..50}
+    do
+        ${psql_command} -c "select 1;"
+        psql_succeeded=$?
+        if [[ ${psql_succeeded} == 0 ]]; then
+            # if ready, return
+            echo "${program_name} database up"
+            return
+        else
+            # if not ready, wait
+            echo "${program_name} waiting for database: ${i}"
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "${program_name} waiting for database: FAILED"
+    exit 1
+}
+
+# Used to synchronize with the test runner
+notify_test_complete () {
+    echo "${program_name} test complete"
+}
+
+does_test_data_exist () {
+    query_result=$(${psql_command} -t -c "select count(*) from test_table_1 where key_column = 'test_key_1';" | head -1 | tr -d ' ')
+    if [[ "${query_result}" == "1" ]]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+create_test_data () {
+    ${psql_command} -c "create table test_table_1 (key_column text, value_column text, primary key(key_column));"
+    ${psql_command} -c "insert into test_table_1 values ('test_key_1', 'test_value_1');"
+}
+
+wait_forever() {
+    # Loop to keep docker/k8s happy since this is the container entrypoint
+    while :; do sleep 600; done
+}
+
+wait_for_database_up
+
+# Check if the test database content exists already
+if does_test_data_exist; then
+    # If so, log saying so. Test harness will look for this log output
+    echo "${program_name} test data already exists"
+else
+    # Otherwise log saying the content was not present
+    echo "${program_name} test data does not exist"
+    echo "${program_name} creating test data"
+    # then create it
+    create_test_data
+fi
+
+notify_test_complete
+wait_forever

stack_orchestrator/data/container-build/cerc-test-database-container/Dockerfile

@@ -0,0 +1,3 @@
+FROM postgres:16-bullseye
+
+EXPOSE 5432

stack_orchestrator/data/container-build/cerc-test-database-container/build.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Build cerc/test-container
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+docker build -t cerc/test-database-container:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR

stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile

@@ -1,6 +1,6 @@
 # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
 # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT=20-bullseye
+ARG VARIANT=20-bullseye-slim
 FROM node:${VARIANT}
 
 ARG USERNAME=node
@@ -24,6 +24,10 @@ RUN \
     && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
     # Install eslint
     && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
+    # Install semver
+    && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
+    # Install pnpm
+    && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
     && npm cache clean --force > /dev/null 2>&1
 
 # [Optional] Uncomment this section to install additional OS packages.
@@ -43,7 +47,6 @@ COPY scripts /scripts
 # RUN su node -c "npm install -g <your-package-list-here>"
 
 RUN mkdir -p /config
-COPY ./config.yml /config
 
 # Install simple web server for now (use nginx perhaps later)
 RUN yarn global add http-server

stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp

@@ -1,11 +1,12 @@
 FROM cerc/webapp-base:local as builder
 
 ARG CERC_BUILD_TOOL
+ARG CERC_BUILD_OUTPUT_DIR
 
 WORKDIR /app
 COPY . .
-RUN rm -rf node_modules build .next*
-RUN /scripts/build-app.sh /app build /data
+RUN rm -rf node_modules build dist .next*
+RUN /scripts/build-app.sh /app /data
 
 FROM cerc/webapp-base:local
 COPY --from=builder /data /data

stack_orchestrator/data/container-build/cerc-webapp-base/config.yml

@@ -1 +0,0 @@
-# Put config here.

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/apply-runtime-env.sh

@@ -18,8 +18,8 @@ if [ -f ".env" ]; then
   rm -f $TMP_ENV
 fi
 
-for f in $(find . -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do
-  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},();]' '\n' | egrep -o -e '^"CERC_RUNTIME_ENV_[^\"]+"' -e '^"LACONIC_HOSTED_CONFIG_[^\"]+"'); do
+for f in $(find . -type f \( -regex '.*.html?' -or -regex ".*.[tj]s\(x\|on\)?$" \) | grep -v 'node_modules' | grep -v '.git'); do
+  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '["/\\{},();]' '\n' | tr -s "[']" '\n' | egrep -o -e '^CERC_RUNTIME_ENV_.+$' -e '^LACONIC_HOSTED_CONFIG_.+$'); do
     orig_name=$(echo -n "${e}" | sed 's/"//g')
     cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
     cur_val=$(echo -n "\$${cur_name}" | envsubst)

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh

@@ -7,30 +7,55 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
-WORK_DIR="${1:-/app}"
-OUTPUT_DIR="${2:-build}"
-DEST_DIR="${3:-/data}"
+CERC_BUILD_OUTPUT_DIR="${CERC_BUILD_OUTPUT_DIR}"
 
-if [ -f "${WORK_DIR}/package.json" ]; then
+WORK_DIR="${1:-/app}"
+DEST_DIR="${2:-/data}"
+
+if [ -f "${WORK_DIR}/build-webapp.sh" ]; then
+  echo "Building webapp with ${WORK_DIR}/build-webapp.sh ..."
+  cd "${WORK_DIR}" || exit 1
+
+  rm -rf "${DEST_DIR}"
+  ./build-webapp.sh "${DEST_DIR}" || exit 1
+elif [ -f "${WORK_DIR}/package.json" ]; then
   echo "Building node-based webapp ..."
   cd "${WORK_DIR}" || exit 1
 
   if [ -z "$CERC_BUILD_TOOL" ]; then
-    if [ -f "yarn.lock" ]; then
+    if [ -f "pnpm-lock.yaml" ]; then
+      CERC_BUILD_TOOL=pnpm
+    elif [ -f "yarn.lock" ]; then
       CERC_BUILD_TOOL=yarn
     else
       CERC_BUILD_TOOL=npm
     fi
   fi
 
-  $CERC_BUILD_TOOL install || exit 1
-  $CERC_BUILD_TOOL build || exit 1
+  time $CERC_BUILD_TOOL install || exit 1
+  time $CERC_BUILD_TOOL build || exit 1
 
   rm -rf "${DEST_DIR}"
-  mv "${WORK_DIR}/${OUTPUT_DIR}" "${DEST_DIR}"
+  if [ -z "${CERC_BUILD_OUTPUT_DIR}" ]; then
+    if [ -d "${WORK_DIR}/dist" ]; then
+      CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/dist"
+    elif [ -d "${WORK_DIR}/build" ]; then
+      CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/build"
+    else
+      echo "ERROR: Unable to locate build output.  Set with --extra-build-args \"--build-arg CERC_BUILD_OUTPUT_DIR=path\"" 1>&2
+      exit 1
+    fi
+  fi
+  mv "${CERC_BUILD_OUTPUT_DIR}" "${DEST_DIR}"
 else
   echo "Copying static app ..."
   mv "${WORK_DIR}" "${DEST_DIR}"
 fi
 
+# One special fix ...
+cd "${DEST_DIR}"
+for f in $(find . -type f -name '*.htm*'); do
+  sed -i -e 's#/LACONIC_HOSTED_CONFIG_homepage/#LACONIC_HOSTED_CONFIG_homepage/#g' "$f"
+done
+
 exit 0

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh

@@ -3,13 +3,40 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
+CERC_LISTEN_PORT=${CERC_LISTEN_PORT:-80}
 CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}"
 CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}"
+CERC_SINGLE_PAGE_APP="${CERC_SINGLE_PAGE_APP}"
+
+if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then
+  if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ] && [ -d "${CERC_WEBAPP_FILES_DIR}/static" ]; then
+    CERC_SINGLE_PAGE_APP=true
+  else
+    CERC_SINGLE_PAGE_APP=false
+  fi
+fi
 
 if [ "true" == "$CERC_ENABLE_CORS" ]; then
   CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --cors"
 fi
 
-/scripts/apply-webapp-config.sh /config/config.yml ${CERC_WEBAPP_FILES_DIR}
+if [ "true" == "$CERC_SINGLE_PAGE_APP" ]; then
+  # Create a catchall redirect back to /
+  CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --proxy http://localhost:${CERC_LISTEN_PORT}?"
+fi
+
+LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
+if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
+  if [ -f "/config/laconic-hosted-config.yml" ]; then
+    LACONIC_HOSTED_CONFIG_FILE="/config/laconic-hosted-config.yml"
+  elif [ -f "/config/config.yml" ]; then
+    LACONIC_HOSTED_CONFIG_FILE="/config/config.yml"
+  fi
+fi
+
+if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
+  /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "${CERC_WEBAPP_FILES_DIR}"
+fi
+
 /scripts/apply-runtime-env.sh ${CERC_WEBAPP_FILES_DIR}
-http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT:-80} ${CERC_WEBAPP_FILES_DIR}
+http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT} "${CERC_WEBAPP_FILES_DIR}"

stack_orchestrator/data/k8s/components/ingress/ingress-nginx-kind-deploy.yaml

@@ -0,0 +1,673 @@
+# from: https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
+# via: https://kind.sigs.k8s.io/docs/user/ingress/#ingress-nginx
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+  name: ingress-nginx
+---
+apiVersion: v1
+automountServiceAccountToken: true
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx
+  namespace: ingress-nginx
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx
+  namespace: ingress-nginx
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - pods
+  - secrets
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses/status
+  verbs:
+  - update
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingressclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - coordination.k8s.io
+  resourceNames:
+  - ingress-nginx-leader
+  resources:
+  - leases
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - list
+  - watch
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - endpoints
+  - nodes
+  - pods
+  - secrets
+  - namespaces
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses/status
+  verbs:
+  - update
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingressclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - list
+  - watch
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+rules:
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - validatingwebhookconfigurations
+  verbs:
+  - get
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx
+  namespace: ingress-nginx
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ingress-nginx
+subjects:
+- kind: ServiceAccount
+  name: ingress-nginx
+  namespace: ingress-nginx
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ingress-nginx-admission
+subjects:
+- kind: ServiceAccount
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ingress-nginx
+subjects:
+- kind: ServiceAccount
+  name: ingress-nginx
+  namespace: ingress-nginx
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ingress-nginx-admission
+subjects:
+- kind: ServiceAccount
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+---
+apiVersion: v1
+data:
+  allow-snippet-annotations: "false"
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+spec:
+  ipFamilies:
+  - IPv4
+  ipFamilyPolicy: SingleStack
+  ports:
+  - appProtocol: http
+    name: http
+    port: 80
+    protocol: TCP
+    targetPort: http
+  - appProtocol: https
+    name: https
+    port: 443
+    protocol: TCP
+    targetPort: https
+  selector:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+  type: NodePort
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-controller-admission
+  namespace: ingress-nginx
+spec:
+  ports:
+  - appProtocol: https
+    name: https-webhook
+    port: 443
+    targetPort: webhook
+  selector:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+spec:
+  minReadySeconds: 0
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: controller
+      app.kubernetes.io/instance: ingress-nginx
+      app.kubernetes.io/name: ingress-nginx
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: controller
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/part-of: ingress-nginx
+        app.kubernetes.io/version: 1.9.6
+    spec:
+      containers:
+      - args:
+        - /nginx-ingress-controller
+        - --election-id=ingress-nginx-leader
+        - --controller-class=k8s.io/ingress-nginx
+        - --ingress-class=nginx
+        - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
+        - --validating-webhook=:8443
+        - --validating-webhook-certificate=/usr/local/certificates/cert
+        - --validating-webhook-key=/usr/local/certificates/key
+        - --watch-ingress-without-class=true
+        - --publish-status-address=localhost
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: LD_PRELOAD
+          value: /usr/local/lib/libmimalloc.so
+        image: registry.k8s.io/ingress-nginx/controller:v1.9.6@sha256:1405cc613bd95b2c6edd8b2a152510ae91c7e62aea4698500d23b2145960ab9c
+        imagePullPolicy: IfNotPresent
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - /wait-shutdown
+        livenessProbe:
+          failureThreshold: 5
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+        name: controller
+        ports:
+        - containerPort: 80
+          hostPort: 80
+          name: http
+          protocol: TCP
+        - containerPort: 443
+          hostPort: 443
+          name: https
+          protocol: TCP
+        - containerPort: 8443
+          name: webhook
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+        resources:
+          requests:
+            cpu: 100m
+            memory: 90Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - ALL
+          readOnlyRootFilesystem: false
+          runAsNonRoot: true
+          runAsUser: 101
+          seccompProfile:
+            type: RuntimeDefault
+        volumeMounts:
+        - mountPath: /usr/local/certificates/
+          name: webhook-cert
+          readOnly: true
+      dnsPolicy: ClusterFirst
+      nodeSelector:
+        ingress-ready: "true"
+        kubernetes.io/os: linux
+      serviceAccountName: ingress-nginx
+      terminationGracePeriodSeconds: 0
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+        operator: Equal
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+        operator: Equal
+      volumes:
+      - name: webhook-cert
+        secret:
+          secretName: ingress-nginx-admission
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission-create
+  namespace: ingress-nginx
+spec:
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: admission-webhook
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/part-of: ingress-nginx
+        app.kubernetes.io/version: 1.9.6
+      name: ingress-nginx-admission-create
+    spec:
+      containers:
+      - args:
+        - create
+        - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc
+        - --namespace=$(POD_NAMESPACE)
+        - --secret-name=ingress-nginx-admission
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20231226-1a7112e06@sha256:25d6a5f11211cc5c3f9f2bf552b585374af287b4debf693cacbe2da47daa5084
+        imagePullPolicy: IfNotPresent
+        name: create
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 65532
+          seccompProfile:
+            type: RuntimeDefault
+      nodeSelector:
+        kubernetes.io/os: linux
+      restartPolicy: OnFailure
+      serviceAccountName: ingress-nginx-admission
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission-patch
+  namespace: ingress-nginx
+spec:
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: admission-webhook
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/part-of: ingress-nginx
+        app.kubernetes.io/version: 1.9.6
+      name: ingress-nginx-admission-patch
+    spec:
+      containers:
+      - args:
+        - patch
+        - --webhook-name=ingress-nginx-admission
+        - --namespace=$(POD_NAMESPACE)
+        - --patch-mutating=false
+        - --secret-name=ingress-nginx-admission
+        - --patch-failure-policy=Fail
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20231226-1a7112e06@sha256:25d6a5f11211cc5c3f9f2bf552b585374af287b4debf693cacbe2da47daa5084
+        imagePullPolicy: IfNotPresent
+        name: patch
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 65532
+          seccompProfile:
+            type: RuntimeDefault
+      nodeSelector:
+        kubernetes.io/os: linux
+      restartPolicy: OnFailure
+      serviceAccountName: ingress-nginx-admission
+---
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: nginx
+spec:
+  controller: k8s.io/ingress-nginx
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: ingress-nginx-controller-admission
+      namespace: ingress-nginx
+      path: /networking/v1/ingresses
+  failurePolicy: Fail
+  matchPolicy: Equivalent
+  name: validate.nginx.ingress.kubernetes.io
+  rules:
+  - apiGroups:
+    - networking.k8s.io
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - ingresses
+  sideEffects: None

stack_orchestrator/data/stacks/container-registry/README.md

@@ -0,0 +1,3 @@
+# Container Registry Stack
+
+Host a container image registry

stack_orchestrator/data/stacks/container-registry/stack.yml

@@ -0,0 +1,5 @@
+version: "1.0"
+name: container-registry
+description: "Container registry stack"
+pods:
+  - container-registry

stack_orchestrator/data/stacks/fixturenet-eth/README.md

@@ -1,6 +1,6 @@
 # fixturenet-eth
 
-Instructions for deploying a local a geth + lighthouse blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator)):
+Instructions for deploying a local a geth + lighthouse blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator (the installation of which is covered [here](https://git.vdb.to/cerc-io/stack-orchestrator)):
 
 ## Clone required repositories
 

stack_orchestrator/data/stacks/fixturenet-laconic-loaded/README.md

@@ -7,11 +7,11 @@ Instructions for deploying a local Laconic blockchain "fixturenet" for developme
 **Note:** For building some NPMs, access to the @lirewine repositories is required. If you don't have access, see [this tutorial](/docs/laconicd-fixturenet.md) to run this stack
 
 ## 1. Install Laconic Stack Orchestrator
-Installation is covered in detail [here](https://github.com/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as:
+Installation is covered in detail [here](https://git.vdb.to/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as:
 ```
 $ mkdir my-working-dir
 $ cd my-working-dir
-$ curl -L -o ./laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+$ curl -L -o ./laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
 $ chmod +x ./laconic-so
 $ export PATH=$PATH:$(pwd)  # Or move laconic-so to ~/bin or your favorite on-path directory
 ```

stack_orchestrator/data/stacks/fixturenet-laconicd/README.md

@@ -3,11 +3,11 @@
 Instructions for deploying a local Laconic blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator.
 
 ## 1. Install Laconic Stack Orchestrator
-Installation is covered in detail [here](https://github.com/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as:
+Installation is covered in detail [here](https://git.vdb.to/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as:
 ```
 $ mkdir my-working-dir
 $ cd my-working-dir
-$ curl -L -o ./laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+$ curl -L -o ./laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
 $ chmod +x ./laconic-so
 $ export PATH=$PATH:$(pwd)  # Or move laconic-so to ~/bin or your favorite on-path directory
 ```

stack_orchestrator/data/stacks/mars-v2/README.md

@@ -0,0 +1,16 @@
+# mars
+
+On a fresh Digital Ocean droplet with Ubuntu:
+
+```
+git clone https://github.com/cerc-io/stack-orchestrator
+cd stack-orchestrator
+./scripts/quick-install-linux.sh
+```
+Read and follow the instructions output from the above output to complete installation, then:
+
+```
+laconic-so --stack mars-v2 setup-repositories
+laconic-so --stack mars-v2 build-containers
+laconic-so --stack mars-v2 deploy up
+```

stack_orchestrator/data/stacks/mars-v2/stack.yml

@@ -0,0 +1,8 @@
+version: "0.1"
+name: mars-v2
+repos:
+  - github.com/mars-protocol/mars-v2-frontend
+containers:
+  - cerc/mars-v2
+pods:
+  - mars-v2

stack_orchestrator/data/stacks/mars/README.md

@@ -0,0 +1,16 @@
+# mars
+
+On a fresh Digital Ocean droplet with Ubuntu:
+
+```
+git clone https://github.com/cerc-io/stack-orchestrator
+cd stack-orchestrator
+./scripts/quick-install-linux.sh
+```
+Read and follow the instructions output from the above output to complete installation, then:
+
+```
+laconic-so --stack mars setup-repositories
+laconic-so --stack mars build-containers
+laconic-so --stack mars deploy up
+```

stack_orchestrator/data/stacks/mars/stack.yml

@@ -0,0 +1,8 @@
+version: "0.1"
+name: mars
+repos:
+  - github.com/cerc-io/mars-interface
+containers:
+  - cerc/mars
+pods:
+  - mars

stack_orchestrator/data/stacks/mobymask/README.md

@@ -4,7 +4,7 @@ The MobyMask watcher is a Laconic Network component that provides efficient acce
 
 ## Deploy the MobyMask Watcher
 
-The instructions below show how to deploy a MobyMask watcher using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator#install)).
+The instructions below show how to deploy a MobyMask watcher using laconic-stack-orchestrator (the installation of which is covered [here](https://git.vdb.to/cerc-io/stack-orchestrator#install)).
 
 This deployment expects that ipld-eth-server's endpoints are available on the local machine at http://ipld-eth-server.example.com:8083/graphql and http://ipld-eth-server.example.com:8082. More advanced configurations are supported by modifying the watcher's [config file](../../config/watcher-mobymask/mobymask-watcher.toml).
 

stack_orchestrator/data/stacks/ping-pub/README.md

@@ -0,0 +1,10 @@
+# ping-pub
+Experimental block explorer for laconic
+
+```
+laconic-so --stack ping-pub setup-repositories
+laconic-so --stack ping-pub build-containers
+laconic-so --stack ping-pub deploy init --output ping-pub-spec.yml --map-ports-to-host localhost-same
+laconic-so --stack ping-pub deploy create --spec-file ping-pub-spec.yml --deployment-dir pp-deployment
+laconic-so deployment --dir pp-deployment start
+```

stack_orchestrator/data/stacks/ping-pub/stack.yml

@@ -0,0 +1,9 @@
+version: "0.1"
+name: ping-pub
+repos:
+  # fork, but only for config & Dockerfile reasonsb
+  - github.com/LaconicNetwork/explorer@laconic
+containers:
+  - cerc/ping-pub
+pods:
+  - ping-pub

stack_orchestrator/data/stacks/snowballtools-base-backend/stack.yml

@@ -0,0 +1,10 @@
+version: "1.0"
+name: snowballtools-base-backend
+description: "snowballtools-base-backend"
+repos:
+  - github.com/snowball-tools/snowballtools-base
+containers:
+  - cerc/webapp-base
+  - cerc/snowballtools-base-backend
+pods:
+  - snowballtools-base-backend

stack_orchestrator/data/stacks/test-database/README.md

@@ -0,0 +1,3 @@
+# Test Database Stack
+
+A stack with a database for test/demo purposes.

stack_orchestrator/data/stacks/test-database/stack.yml

@@ -0,0 +1,9 @@
+version: "1.0"
+name: test
+description: "A test database stack"
+repos:
+containers:
+  - cerc/test-database-container
+  - cerc/test-database-client
+pods:
+  - test-database

stack_orchestrator/data/stacks/uniswap-urbit-app/README.md

@@ -33,23 +33,29 @@ laconic-so --stack uniswap-urbit-app deploy init --output uniswap-urbit-app-spec
 
 ### Ports
 
-Edit `network` in spec file to map container ports to same ports in host:
+Edit `uniswap-urbit-app-spec.yml` such that it looks like:
 
 ```yml
-...
+stack: uniswap-urbit-app
+deploy-to: compose
 network:
   ports:
-    urbit-fake-ship:
-      - '8080:80'
     proxy-server:
       - '4000:4000'
+    urbit-fake-ship:
+      - '8080:80'
     ipfs:
-      - '8081:8080'
-      - '5001:5001'
-...
+     - '4001'
+     - '8081:8080'
+     - 0.0.0.0:5001:5001
+volumes:
+  urbit_app_builds: ./data/urbit_app_builds
+  urbit_data: ./data/urbit_data
+  ipfs-import: ./data/ipfs-import
+  ipfs-data: ./data/ipfs-data
 ```
 
-Note: Skip the `ipfs` ports if need to use an externally running IPFS node
+Note: Skip the `ipfs` ports if using an externally running IPFS node, set via `config.env`, below.
 
 ### Data volumes
 

stack_orchestrator/data/stacks/webapp-deployer-backend/stack.yml

@@ -2,10 +2,10 @@ version: "1.0"
 name: webapp-deployer-backend
 description: "Deployer for webapps"
 repos:
-  - git.vdb.to:telackey/webapp-deployment-status-api
+  - git.vdb.to/telackey/webapp-deployment-status-api
 containers:
   - cerc/webapp-deployer-backend
 pods:
   - name: webapp-deployer-backend
-    repository: git.vdb.to:telackey/webapp-deployment-status-api
+    repository: git.vdb.to/telackey/webapp-deployment-status-api
     path: ./

stack_orchestrator/deploy/deployment_create.py

@@ -27,6 +27,7 @@ from stack_orchestrator.opts import opts
 from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
                                      global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
                                      get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file)
+from stack_orchestrator.deploy.spec import Spec
 from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand
 from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
@@ -111,6 +112,7 @@ def _create_bind_dir_if_relative(volume, path_string, compose_dir):
 
 # See: https://stackoverflow.com/questions/45699189/editing-docker-compose-yml-with-pyyaml
 def _fixup_pod_file(pod, spec, compose_dir):
+    deployment_type = spec[constants.deploy_to_key]
     # Fix up volumes
     if "volumes" in spec:
         spec_volumes = spec["volumes"]
@@ -119,27 +121,34 @@ def _fixup_pod_file(pod, spec, compose_dir):
             for volume in pod_volumes.keys():
                 if volume in spec_volumes:
                     volume_spec = spec_volumes[volume]
-                    volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}"
-                    _create_bind_dir_if_relative(volume, volume_spec, compose_dir)
-                    new_volume_spec = {"driver": "local",
-                                       "driver_opts": {
-                                           "type": "none",
-                                           "device": volume_spec_fixedup,
-                                           "o": "bind"
-                                       }
-                                       }
-                    pod["volumes"][volume] = new_volume_spec
+                    if volume_spec:
+                        volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}"
+                        _create_bind_dir_if_relative(volume, volume_spec, compose_dir)
+                        # this is Docker specific
+                        if spec.is_docker_deployment():
+                            new_volume_spec = {
+                                "driver": "local",
+                                "driver_opts": {
+                                    "type": "none",
+                                    "device": volume_spec_fixedup,
+                                    "o": "bind"
+                                }
+                            }
+                            pod["volumes"][volume] = new_volume_spec
 
     # Fix up configmaps
-    if "configmaps" in spec:
-        spec_cfgmaps = spec["configmaps"]
-        if "volumes" in pod:
-            pod_volumes = pod["volumes"]
-            for volume in pod_volumes.keys():
-                if volume in spec_cfgmaps:
-                    volume_cfg = spec_cfgmaps[volume]
-                    # Just make the dir (if necessary)
-                    _create_bind_dir_if_relative(volume, volume_cfg, compose_dir)
+    if constants.configmaps_key in spec:
+        if spec.is_kubernetes_deployment():
+            spec_cfgmaps = spec[constants.configmaps_key]
+            if "volumes" in pod:
+                pod_volumes = pod[constants.volumes_key]
+                for volume in pod_volumes.keys():
+                    if volume in spec_cfgmaps:
+                        volume_cfg = spec_cfgmaps[volume]
+                        # Just make the dir (if necessary)
+                        _create_bind_dir_if_relative(volume, volume_cfg, compose_dir)
+        else:
+            print(f"Warning: ConfigMaps not supported for {deployment_type}")
 
     # Fix up ports
     if "network" in spec and "ports" in spec["network"]:
@@ -318,12 +327,14 @@ def init_operation(deploy_command_context, stack, deployer_type, config,
     default_spec_file_content = call_stack_deploy_init(deploy_command_context)
     spec_file_content = {"stack": stack, constants.deploy_to_key: deployer_type}
     if deployer_type == "k8s":
-        if kube_config is None:
+        if kube_config:
+            spec_file_content.update({constants.kube_config_key: kube_config})
+        else:
             error_exit("--kube-config must be supplied with --deploy-to k8s")
-        if image_registry is None:
-            error_exit("--image-registry must be supplied with --deploy-to k8s")
-        spec_file_content.update({constants.kube_config_key: kube_config})
-        spec_file_content.update({constants.image_resigtry_key: image_registry})
+        if image_registry:
+            spec_file_content.update({constants.image_registry_key: image_registry})
+        else:
+            print("WARNING: --image-registry not specified, only default container registries (eg, Docker Hub) will be available")
     else:
         # Check for --kube-config supplied for non-relevant deployer types
         if kube_config is not None:
@@ -358,10 +369,16 @@ def init_operation(deploy_command_context, stack, deployer_type, config,
         volume_descriptors = {}
         configmap_descriptors = {}
         for named_volume in named_volumes["rw"]:
-            volume_descriptors[named_volume] = f"./data/{named_volume}"
+            if "k8s" in deployer_type:
+                volume_descriptors[named_volume] = None
+            else:
+                volume_descriptors[named_volume] = f"./data/{named_volume}"
         for named_volume in named_volumes["ro"]:
-            if "k8s" in deployer_type and "config" in named_volume:
-                configmap_descriptors[named_volume] = f"./data/{named_volume}"
+            if "k8s" in deployer_type:
+                if "config" in named_volume:
+                    configmap_descriptors[named_volume] = f"./configmaps/{named_volume}"
+                else:
+                    volume_descriptors[named_volume] = None
             else:
                 volume_descriptors[named_volume] = f"./data/{named_volume}"
         if volume_descriptors:
@@ -406,6 +423,17 @@ def _create_deployment_file(deployment_dir: Path):
         output_file.write(f"{constants.cluster_id_key}: {cluster}\n")
 
 
+def _check_volume_definitions(spec):
+    if spec.is_kubernetes_deployment():
+        for volume_name, volume_path in spec.get_volumes().items():
+            if volume_path:
+                if not os.path.isabs(volume_path):
+                    raise Exception(
+                        f"Relative path {volume_path} for volume {volume_name} not "
+                        f"supported for deployment type {spec.get_deployment_type()}"
+                    )
+
+
 @click.command()
 @click.option("--spec-file", required=True, help="Spec file to use to create this deployment")
 @click.option("--deployment-dir", help="Create deployment files in this directory")
@@ -421,7 +449,8 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
 # The init command's implementation is in a separate function so that we can
 # call it from other commands, bypassing the click decoration stuff
 def create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers):
-    parsed_spec = get_parsed_deployment_spec(spec_file)
+    parsed_spec = Spec(os.path.abspath(spec_file), get_parsed_deployment_spec(spec_file))
+    _check_volume_definitions(parsed_spec)
     stack_name = parsed_spec["stack"]
     deployment_type = parsed_spec[constants.deploy_to_key]
     stack_file = get_stack_file_path(stack_name)

stack_orchestrator/deploy/images.py

@@ -29,6 +29,29 @@ def _image_needs_pushed(image: str):
     return image.endswith(":local")
 
 
+def remote_image_exists(remote_repo_url: str, local_tag: str):
+    docker = DockerClient()
+    try:
+        remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
+        result = docker.manifest.inspect(remote_tag)
+        return True if result else False
+    except Exception:  # noqa: E722
+        return False
+
+
+def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
+    if not additional_tags:
+        return
+
+    if not remote_image_exists(remote_repo_url, local_tag):
+        raise Exception(f"{local_tag} does not exist in {remote_repo_url}")
+
+    docker = DockerClient()
+    remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
+    new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
+    docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags)
+
+
 def remote_tag_for_image(image: str, remote_repo_url: str):
     # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
     major_parts = image.split("/", 2)
@@ -46,7 +69,7 @@ def push_images_operation(command_context: DeployCommandContext, deployment_cont
     cluster_context = command_context.cluster_context
     images: Set[str] = images_for_deployment(cluster_context.compose_files)
     # Tag the images for the remote repo
-    remote_repo_url = deployment_context.spec.obj[constants.image_resigtry_key]
+    remote_repo_url = deployment_context.spec.obj[constants.image_registry_key]
     docker = DockerClient()
     for image in images:
         if _image_needs_pushed(image):

stack_orchestrator/deploy/k8s/cluster_info.py

@@ -21,13 +21,42 @@ from typing import Any, List, Set
 from stack_orchestrator.opts import opts
 from stack_orchestrator.util import env_var_map_from_file
 from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files
-from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path
-from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map
+from stack_orchestrator.deploy.k8s.helpers import get_kind_pv_bind_mount_path
+from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map, envs_from_compose_file, merge_envs
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment
 from stack_orchestrator.deploy.deploy_types import DeployEnvVars
-from stack_orchestrator.deploy.spec import Spec
+from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits
 from stack_orchestrator.deploy.images import remote_tag_for_image
 
+DEFAULT_VOLUME_RESOURCES = Resources({
+    "reservations": {"storage": "2Gi"}
+})
+
+DEFAULT_CONTAINER_RESOURCES = Resources({
+    "reservations": {"cpus": "0.1", "memory": "200M"},
+    "limits": {"cpus": "1.0", "memory": "2000M"},
+})
+
+
+def to_k8s_resource_requirements(resources: Resources) -> client.V1ResourceRequirements:
+    def to_dict(limits: ResourceLimits):
+        if not limits:
+            return None
+
+        ret = {}
+        if limits.cpus:
+            ret["cpu"] = str(limits.cpus)
+        if limits.memory:
+            ret["memory"] = f"{int(limits.memory / (1000 * 1000))}M"
+        if limits.storage:
+            ret["storage"] = f"{int(limits.storage / (1000 * 1000))}M"
+        return ret
+
+    return client.V1ResourceRequirements(
+        requests=to_dict(resources.reservations),
+        limits=to_dict(resources.limits)
+    )
+
 
 class ClusterInfo:
     parsed_pod_yaml_map: Any
@@ -49,7 +78,30 @@ class ClusterInfo:
         if (opts.o.debug):
             print(f"Env vars: {self.environment_variables.map}")
 
-    def get_ingress(self):
+    def get_nodeport(self):
+        for pod_name in self.parsed_pod_yaml_map:
+            pod = self.parsed_pod_yaml_map[pod_name]
+            services = pod["services"]
+            for service_name in services:
+                service_info = services[service_name]
+                if "ports" in service_info:
+                    port = int(service_info["ports"][0])
+                    if opts.o.debug:
+                        print(f"service port: {port}")
+        service = client.V1Service(
+            metadata=client.V1ObjectMeta(name=f"{self.app_name}-nodeport"),
+            spec=client.V1ServiceSpec(
+                type="NodePort",
+                ports=[client.V1ServicePort(
+                    port=port,
+                    target_port=port
+                )],
+                selector={"app": self.app_name}
+            )
+        )
+        return service
+
+    def get_ingress(self, use_tls=False):
         # No ingress for a deployment that has no http-proxy defined, for now
         http_proxy_info_list = self.spec.get_http_proxy()
         ingress = None
@@ -58,13 +110,32 @@ class ClusterInfo:
             http_proxy_info = http_proxy_info_list[0]
             if opts.o.debug:
                 print(f"http-proxy: {http_proxy_info}")
-            # TODO: good enough parsing for webapp deployment for now
+
             host_name = http_proxy_info["host-name"]
+
+            tls = None
+            tls_issuer = None
+
+            if use_tls:
+                tls_info = http_proxy_info.get("tls", {})
+                tls_hosts = tls_info.get("hosts", [host_name])
+                tls_issuer = tls_info.get("issuer", "letsencrypt-prod")
+                tls_secret_name = f"{self.app_name}-tls"
+                if "secret" in tls_info:
+                    # If an existing secret is specified, unset the issuer so that we don't try to re-request it.
+                    tls_secret_name = tls_info["secret"]
+                    tls_issuer = None
+
+                if opts.o.debug:
+                    print(f"TLS hosts/secret: {tls_hosts}/{tls_secret_name}")
+
+                tls = [client.V1IngressTLS(
+                    hosts=tls_hosts,
+                    secret_name=tls_secret_name
+                )]
+
+            # TODO: good enough parsing for webapp deployment for now
             rules = []
-            tls = [client.V1IngressTLS(
-                hosts=[host_name],
-                secret_name=f"{self.app_name}-tls"
-            )]
             paths = []
             for route in http_proxy_info["routes"]:
                 path = route["path"]
@@ -95,13 +166,15 @@ class ClusterInfo:
                 tls=tls,
                 rules=rules
             )
+            annotations = {
+                "kubernetes.io/ingress.class": "nginx",
+            }
+            if tls_issuer:
+                annotations["cert-manager.io/cluster-issuer"] = tls_issuer
             ingress = client.V1Ingress(
                 metadata=client.V1ObjectMeta(
                     name=f"{self.app_name}-ingress",
-                    annotations={
-                        "kubernetes.io/ingress.class": "nginx",
-                        "cert-manager.io/cluster-issuer": "letsencrypt-prod"
-                    }
+                    annotations=annotations
                 ),
                 spec=spec
             )
@@ -135,26 +208,40 @@ class ClusterInfo:
         result = []
         spec_volumes = self.spec.get_volumes()
         named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        resources = self.spec.get_volume_resources()
+        if not resources:
+            resources = DEFAULT_VOLUME_RESOURCES
         if opts.o.debug:
             print(f"Spec Volumes: {spec_volumes}")
             print(f"Named Volumes: {named_volumes}")
-        for volume_name in spec_volumes:
+            print(f"Resources: {resources}")
+        for volume_name, volume_path in spec_volumes.items():
             if volume_name not in named_volumes:
                 if opts.o.debug:
                     print(f"{volume_name} not in pod files")
                 continue
+
+            labels = {
+                "app": self.app_name,
+                "volume-label": f"{self.app_name}-{volume_name}"
+            }
+            if volume_path:
+                storage_class_name = "manual"
+                k8s_volume_name = f"{self.app_name}-{volume_name}"
+            else:
+                # These will be auto-assigned.
+                storage_class_name = None
+                k8s_volume_name = None
+
             spec = client.V1PersistentVolumeClaimSpec(
                 access_modes=["ReadWriteOnce"],
-                storage_class_name="manual",
-                resources=client.V1ResourceRequirements(
-                    requests={"storage": "2Gi"}
-                ),
-                volume_name=f"{self.app_name}-{volume_name}"
+                storage_class_name=storage_class_name,
+                resources=to_k8s_resource_requirements(resources),
+                volume_name=k8s_volume_name
             )
             pvc = client.V1PersistentVolumeClaim(
-                metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}",
-                                             labels={"volume-label": f"{self.app_name}-{volume_name}"}),
-                spec=spec,
+                metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}", labels=labels),
+                spec=spec
             )
             result.append(pvc)
         return result
@@ -192,16 +279,35 @@ class ClusterInfo:
         result = []
         spec_volumes = self.spec.get_volumes()
         named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
-        for volume_name in spec_volumes:
+        resources = self.spec.get_volume_resources()
+        if not resources:
+            resources = DEFAULT_VOLUME_RESOURCES
+        for volume_name, volume_path in spec_volumes.items():
+            # We only need to create a volume if it is fully qualified HostPath.
+            # Otherwise, we create the PVC and expect the node to allocate the volume for us.
+            if not volume_path:
+                if opts.o.debug:
+                    print(f"{volume_name} does not require an explicit PersistentVolume, since it is not a bind-mount.")
+                continue
+
             if volume_name not in named_volumes:
                 if opts.o.debug:
                     print(f"{volume_name} not in pod files")
                 continue
+
+            if not os.path.isabs(volume_path):
+                print(f"WARNING: {volume_name}:{volume_path} is not absolute, cannot bind volume.")
+                continue
+
+            if self.spec.is_kind_deployment():
+                host_path = client.V1HostPathVolumeSource(path=get_kind_pv_bind_mount_path(volume_name))
+            else:
+                host_path = client.V1HostPathVolumeSource(path=volume_path)
             spec = client.V1PersistentVolumeSpec(
                 storage_class_name="manual",
                 access_modes=["ReadWriteOnce"],
-                capacity={"storage": "2Gi"},
-                host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name))
+                capacity=to_k8s_resource_requirements(resources).requests,
+                host_path=host_path
             )
             pv = client.V1PersistentVolume(
                 metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}",
@@ -214,6 +320,9 @@ class ClusterInfo:
     # TODO: put things like image pull policy into an object-scope struct
     def get_deployment(self, image_pull_policy: str = None):
         containers = []
+        resources = self.spec.get_container_resources()
+        if not resources:
+            resources = DEFAULT_CONTAINER_RESOURCES
         for pod_name in self.parsed_pod_yaml_map:
             pod = self.parsed_pod_yaml_map[pod_name]
             services = pod["services"]
@@ -226,6 +335,13 @@ class ClusterInfo:
                     if opts.o.debug:
                         print(f"image: {image}")
                         print(f"service port: {port}")
+                merged_envs = merge_envs(
+                    envs_from_compose_file(
+                        service_info["environment"]), self.environment_variables.map
+                ) if "environment" in service_info else self.environment_variables.map
+                envs = envs_from_environment_variables_map(merged_envs)
+                if opts.o.debug:
+                    print(f"Merged envs: {envs}")
                 # Re-write the image tag for remote deployment
                 image_to_use = remote_tag_for_image(
                     image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
@@ -234,19 +350,40 @@ class ClusterInfo:
                     name=container_name,
                     image=image_to_use,
                     image_pull_policy=image_pull_policy,
-                    env=envs_from_environment_variables_map(self.environment_variables.map),
+                    env=envs,
                     ports=[client.V1ContainerPort(container_port=port)],
                     volume_mounts=volume_mounts,
-                    resources=client.V1ResourceRequirements(
-                        requests={"cpu": "100m", "memory": "200Mi"},
-                        limits={"cpu": "1000m", "memory": "2000Mi"},
+                    security_context=client.V1SecurityContext(
+                        privileged=self.spec.get_privileged(),
+                        capabilities=client.V1Capabilities(
+                            add=self.spec.get_capabilities()
+                        ) if self.spec.get_capabilities() else None
                     ),
+                    resources=to_k8s_resource_requirements(resources),
                 )
                 containers.append(container)
         volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec, self.app_name)
         image_pull_secrets = [client.V1LocalObjectReference(name="laconic-registry")]
+
+        annotations = None
+        labels = {"app": self.app_name}
+
+        if self.spec.get_annotations():
+            annotations = {}
+            for key, value in self.spec.get_annotations().items():
+                for service_name in services:
+                    annotations[key.replace("{name}", service_name)] = value
+
+        if self.spec.get_labels():
+            for key, value in self.spec.get_labels().items():
+                for service_name in services:
+                    labels[key.replace("{name}", service_name)] = value
+
         template = client.V1PodTemplateSpec(
-            metadata=client.V1ObjectMeta(labels={"app": self.app_name}),
+            metadata=client.V1ObjectMeta(
+                annotations=annotations,
+                labels=labels
+            ),
             spec=client.V1PodSpec(containers=containers, image_pull_secrets=image_pull_secrets, volumes=volumes),
         )
         spec = client.V1DeploymentSpec(

stack_orchestrator/deploy/k8s/deploy_k8s.py

@@ -20,7 +20,9 @@ from kubernetes import client, config
 from stack_orchestrator import constants
 from stack_orchestrator.deploy.deployer import Deployer, DeployerConfigGenerator
 from stack_orchestrator.deploy.k8s.helpers import create_cluster, destroy_cluster, load_images_into_kind
-from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, log_stream_from_string, generate_kind_config
+from stack_orchestrator.deploy.k8s.helpers import install_ingress_for_kind, wait_for_ingress_in_kind
+from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, containers_in_pod, log_stream_from_string
+from stack_orchestrator.deploy.k8s.helpers import generate_kind_config
 from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo
 from stack_orchestrator.opts import opts
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
@@ -88,6 +90,16 @@ class K8sDeployer(Deployer):
             if opts.o.debug:
                 print(f"Sending this pv: {pv}")
             if not opts.o.dry_run:
+                try:
+                    pv_resp = self.core_api.read_persistent_volume(name=pv.metadata.name)
+                    if pv_resp:
+                        if opts.o.debug:
+                            print("PVs already present:")
+                            print(f"{pv_resp}")
+                        continue
+                except:  # noqa: E722
+                    pass
+
                 pv_resp = self.core_api.create_persistent_volume(body=pv)
                 if opts.o.debug:
                     print("PVs created:")
@@ -100,6 +112,17 @@ class K8sDeployer(Deployer):
                 print(f"Sending this pvc: {pvc}")
 
             if not opts.o.dry_run:
+                try:
+                    pvc_resp = self.core_api.read_namespaced_persistent_volume_claim(
+                        name=pvc.metadata.name, namespace=self.k8s_namespace)
+                    if pvc_resp:
+                        if opts.o.debug:
+                            print("PVCs already present:")
+                            print(f"{pvc_resp}")
+                        continue
+                except:  # noqa: E722
+                    pass
+
                 pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace=self.k8s_namespace)
                 if opts.o.debug:
                     print("PVCs created:")
@@ -154,60 +177,80 @@ class K8sDeployer(Deployer):
                 # Ensure the referenced containers are copied into kind
                 load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set)
             self.connect_api()
+            if self.is_kind():
+                # Now configure an ingress controller (not installed by default in kind)
+                install_ingress_for_kind()
+                # Wait for ingress to start (deployment provisioning will fail unless this is done)
+                wait_for_ingress_in_kind()
+
         else:
             print("Dry run mode enabled, skipping k8s API connect")
 
         self._create_volume_data()
         self._create_deployment()
 
-        if not self.is_kind():
-            ingress: client.V1Ingress = self.cluster_info.get_ingress()
+        # Note: at present we don't support tls for kind (and enabling tls causes errors)
+        ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=not self.is_kind())
+        if ingress:
+            if opts.o.debug:
+                print(f"Sending this ingress: {ingress}")
+            if not opts.o.dry_run:
+                ingress_resp = self.networking_api.create_namespaced_ingress(
+                    namespace=self.k8s_namespace,
+                    body=ingress
+                )
+                if opts.o.debug:
+                    print("Ingress created:")
+                    print(f"{ingress_resp}")
+        else:
+            if opts.o.debug:
+                print("No ingress configured")
 
-            if ingress:
+        nodeport: client.V1Service = self.cluster_info.get_nodeport()
+        if nodeport:
+            if opts.o.debug:
+                print(f"Sending this nodeport: {nodeport}")
+            if not opts.o.dry_run:
+                nodeport_resp = self.core_api.create_namespaced_service(
+                    namespace=self.k8s_namespace,
+                    body=nodeport
+                )
                 if opts.o.debug:
-                    print(f"Sending this ingress: {ingress}")
-                if not opts.o.dry_run:
-                    ingress_resp = self.networking_api.create_namespaced_ingress(
-                        namespace=self.k8s_namespace,
-                        body=ingress
-                    )
-                    if opts.o.debug:
-                        print("Ingress created:")
-                        print(f"{ingress_resp}")
-            else:
-                if opts.o.debug:
-                    print("No ingress configured")
+                    print("NodePort created:")
+                    print(f"{nodeport_resp}")
 
     def down(self, timeout, volumes):  # noqa: C901
         self.connect_api()
         # Delete the k8s objects
-        # Create the host-path-mounted PVs for this deployment
-        pvs = self.cluster_info.get_pvs()
-        for pv in pvs:
-            if opts.o.debug:
-                print(f"Deleting this pv: {pv}")
-            try:
-                pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name)
-                if opts.o.debug:
-                    print("PV deleted:")
-                    print(f"{pv_resp}")
-            except client.exceptions.ApiException as e:
-                _check_delete_exception(e)
 
-        # Figure out the PVCs for this deployment
-        pvcs = self.cluster_info.get_pvcs()
-        for pvc in pvcs:
-            if opts.o.debug:
-                print(f"Deleting this pvc: {pvc}")
-            try:
-                pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim(
-                    name=pvc.metadata.name, namespace=self.k8s_namespace
-                )
+        if volumes:
+            # Create the host-path-mounted PVs for this deployment
+            pvs = self.cluster_info.get_pvs()
+            for pv in pvs:
                 if opts.o.debug:
-                    print("PVCs deleted:")
-                    print(f"{pvc_resp}")
-            except client.exceptions.ApiException as e:
-                _check_delete_exception(e)
+                    print(f"Deleting this pv: {pv}")
+                try:
+                    pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name)
+                    if opts.o.debug:
+                        print("PV deleted:")
+                        print(f"{pv_resp}")
+                except client.exceptions.ApiException as e:
+                    _check_delete_exception(e)
+
+            # Figure out the PVCs for this deployment
+            pvcs = self.cluster_info.get_pvcs()
+            for pvc in pvcs:
+                if opts.o.debug:
+                    print(f"Deleting this pvc: {pvc}")
+                try:
+                    pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim(
+                        name=pvc.metadata.name, namespace=self.k8s_namespace
+                    )
+                    if opts.o.debug:
+                        print("PVCs deleted:")
+                        print(f"{pvc_resp}")
+                except client.exceptions.ApiException as e:
+                    _check_delete_exception(e)
 
         # Figure out the ConfigMaps for this deployment
         cfg_maps = self.cluster_info.get_configmaps()
@@ -245,20 +288,34 @@ class K8sDeployer(Deployer):
         except client.exceptions.ApiException as e:
             _check_delete_exception(e)
 
-        if not self.is_kind():
-            ingress: client.V1Ingress = self.cluster_info.get_ingress()
-            if ingress:
-                if opts.o.debug:
-                    print(f"Deleting this ingress: {ingress}")
-                try:
-                    self.networking_api.delete_namespaced_ingress(
-                        name=ingress.metadata.name, namespace=self.k8s_namespace
-                    )
-                except client.exceptions.ApiException as e:
-                    _check_delete_exception(e)
-            else:
-                if opts.o.debug:
-                    print("No ingress to delete")
+        ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=not self.is_kind())
+        if ingress:
+            if opts.o.debug:
+                print(f"Deleting this ingress: {ingress}")
+            try:
+                self.networking_api.delete_namespaced_ingress(
+                    name=ingress.metadata.name, namespace=self.k8s_namespace
+                )
+            except client.exceptions.ApiException as e:
+                _check_delete_exception(e)
+        else:
+            if opts.o.debug:
+                print("No ingress to delete")
+
+        nodeport: client.V1Service = self.cluster_info.get_nodeport()
+        if nodeport:
+            if opts.o.debug:
+                print(f"Deleting this nodeport: {ingress}")
+            try:
+                self.core_api.delete_namespaced_service(
+                    namespace=self.k8s_namespace,
+                    name=nodeport.metadata.name
+                )
+            except client.exceptions.ApiException as e:
+                _check_delete_exception(e)
+        else:
+            if opts.o.debug:
+                print("No nodeport to delete")
 
         if self.is_kind():
             # Destroy the kind cluster
@@ -293,9 +350,10 @@ class K8sDeployer(Deployer):
                 name=ingress.spec.tls[0].secret_name
             )
 
-            hostname = ingress.spec.tls[0].hosts[0]
+            hostname = ingress.spec.rules[0].host
             ip = ingress.status.load_balancer.ingress[0].ip
-            tls = "notBefore: %s, notAfter: %s" % (cert["status"]["notBefore"], cert["status"]["notAfter"])
+            tls = "notBefore: %s; notAfter: %s; names: %s" % (
+                cert["status"]["notBefore"], cert["status"]["notAfter"], ", ".join(ingress.spec.tls[0].hosts))
         except:  # noqa: E722
             pass
 
@@ -359,9 +417,15 @@ class K8sDeployer(Deployer):
             log_data = "******* Pods not running ********\n"
         else:
             k8s_pod_name = pods[0]
+            containers = containers_in_pod(self.core_api, k8s_pod_name)
             # If the pod is not yet started, the logs request below will throw an exception
             try:
-                log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
+                log_data = ""
+                for container in containers:
+                    container_log = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container=container)
+                    container_log_lines = container_log.splitlines()
+                    for line in container_log_lines:
+                        log_data += f"{container}: {line}\n"
             except client.exceptions.ApiException as e:
                 if opts.o.debug:
                     print(f"Error from read_namespaced_pod_log: {e}")

stack_orchestrator/deploy/k8s/helpers.py

@@ -13,12 +13,14 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from kubernetes import client
+from kubernetes import client, utils, watch
 import os
 from pathlib import Path
 import subprocess
+import re
 from typing import Set, Mapping, List
 
+from stack_orchestrator.util import get_k8s_dir, error_exit
 from stack_orchestrator.opts import opts
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names
 from stack_orchestrator.deploy.deployer import DeployerException
@@ -43,6 +45,33 @@ def destroy_cluster(name: str):
     _run_command(f"kind delete cluster --name {name}")
 
 
+def wait_for_ingress_in_kind():
+    core_v1 = client.CoreV1Api()
+    for i in range(20):
+        warned_waiting = False
+        w = watch.Watch()
+        for event in w.stream(func=core_v1.list_namespaced_pod,
+                              namespace="ingress-nginx",
+                              label_selector="app.kubernetes.io/component=controller",
+                              timeout_seconds=30):
+            if event['object'].status.container_statuses:
+                if event['object'].status.container_statuses[0].ready is True:
+                    if warned_waiting:
+                        print("Ingress controller is ready")
+                    return
+            print("Waiting for ingress controller to become ready...")
+            warned_waiting = True
+    error_exit("ERROR: Timed out waiting for ingress to become ready")
+
+
+def install_ingress_for_kind():
+    api_client = client.ApiClient()
+    ingress_install = os.path.abspath(get_k8s_dir().joinpath("components", "ingress", "ingress-nginx-kind-deploy.yaml"))
+    if opts.o.debug:
+        print("Installing nginx ingress controller in kind cluster")
+    utils.create_from_yaml(api_client, yaml_file=ingress_install)
+
+
 def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]):
     for image in image_set:
         result = _run_command(f"kind load docker-image {image} --name {kind_cluster_name}")
@@ -61,6 +90,17 @@ def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
     return pods
 
 
+def containers_in_pod(core_api: client.CoreV1Api, pod_name: str):
+    containers = []
+    pod_response = core_api.read_namespaced_pod(pod_name, namespace="default")
+    if opts.o.debug:
+        print(f"pod_response: {pod_response}")
+    pod_containers = pod_response.spec.containers
+    for pod_container in pod_containers:
+        containers.append(pod_container.name)
+    return containers
+
+
 def log_stream_from_string(s: str):
     # Note response has to be UTF-8 encoded because the caller expects to decode it
     yield ("ignore", s.encode())
@@ -80,7 +120,7 @@ def named_volumes_from_pod_files(parsed_pod_files):
     return named_volumes
 
 
-def get_node_pv_mount_path(volume_name: str):
+def get_kind_pv_bind_mount_path(volume_name: str):
     return f"/mnt/{volume_name}"
 
 
@@ -105,11 +145,14 @@ def volume_mounts_for_service(parsed_pod_files, service):
                             mount_path = mount_split[1]
                             mount_options = mount_split[2] if len(mount_split) == 3 else None
                             if opts.o.debug:
-                                print(f"volumne_name: {volume_name}")
+                                print(f"volume_name: {volume_name}")
                                 print(f"mount path: {mount_path}")
                                 print(f"mount options: {mount_options}")
                             volume_device = client.V1VolumeMount(
-                                mount_path=mount_path, name=volume_name, read_only="ro" == mount_options)
+                                mount_path=mount_path,
+                                name=volume_name,
+                                read_only="ro" == mount_options
+                            )
                             result.append(volume_device)
     return result
 
@@ -132,18 +175,8 @@ def volumes_for_pod_files(parsed_pod_files, spec, app_name):
     return result
 
 
-def _get_host_paths_for_volumes(parsed_pod_files):
-    result = {}
-    for pod in parsed_pod_files:
-        parsed_pod_file = parsed_pod_files[pod]
-        if "volumes" in parsed_pod_file:
-            volumes = parsed_pod_file["volumes"]
-            for volume_name in volumes.keys():
-                volume_definition = volumes[volume_name]
-                if volume_definition and "driver_opts" in volume_definition:
-                    host_path = volume_definition["driver_opts"]["device"]
-                    result[volume_name] = host_path
-    return result
+def _get_host_paths_for_volumes(deployment_context):
+    return deployment_context.spec.get_volumes()
 
 
 def _make_absolute_host_path(data_mount_path: Path, deployment_dir: Path) -> Path:
@@ -151,12 +184,12 @@ def _make_absolute_host_path(data_mount_path: Path, deployment_dir: Path) -> Pat
         return data_mount_path
     else:
         # Python Path voodo that looks pretty odd:
-        return Path.cwd().joinpath(deployment_dir.joinpath("compose").joinpath(data_mount_path)).resolve()
+        return Path.cwd().joinpath(deployment_dir.joinpath(data_mount_path)).resolve()
 
 
 def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context):
     volume_definitions = []
-    volume_host_path_map = _get_host_paths_for_volumes(parsed_pod_files)
+    volume_host_path_map = _get_host_paths_for_volumes(deployment_context)
     # Note these paths are relative to the location of the pod files (at present)
     # So we need to fix up to make them correct and absolute because kind assumes
     # relative to the cwd.
@@ -176,14 +209,15 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context):
                         volume_name = mount_split[0]
                         mount_path = mount_split[1]
                         if opts.o.debug:
-                            print(f"volumne_name: {volume_name}")
+                            print(f"volume_name: {volume_name}")
                             print(f"map: {volume_host_path_map}")
                             print(f"mount path: {mount_path}")
                         if volume_name not in deployment_context.spec.get_configmaps():
-                            volume_definitions.append(
-                                f"  - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n"
-                                f"    containerPath: {get_node_pv_mount_path(volume_name)}\n"
-                            )
+                            if volume_host_path_map[volume_name]:
+                                volume_definitions.append(
+                                    f"  - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n"
+                                    f"    containerPath: {get_kind_pv_bind_mount_path(volume_name)}\n"
+                                )
     return (
         "" if len(volume_definitions) == 0 else (
             "  extraMounts:\n"
@@ -192,7 +226,8 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir, deployment_context):
     )
 
 
-def _generate_kind_port_mappings(parsed_pod_files):
+# TODO: decide if we need this functionality
+def _generate_kind_port_mappings_from_services(parsed_pod_files):
     port_definitions = []
     for pod in parsed_pod_files:
         parsed_pod_file = parsed_pod_files[pod]
@@ -214,6 +249,46 @@ def _generate_kind_port_mappings(parsed_pod_files):
     )
 
 
+def _generate_kind_port_mappings(parsed_pod_files):
+    port_definitions = []
+    # For now we just map port 80 for the nginx ingress controller we install in kind
+    port_string = "80"
+    port_definitions.append(f"  - containerPort: {port_string}\n    hostPort: {port_string}\n")
+    return (
+        "" if len(port_definitions) == 0 else (
+            "  extraPortMappings:\n"
+            f"{''.join(port_definitions)}"
+        )
+    )
+
+
+# Note: this makes any duplicate definition in b overwrite a
+def merge_envs(a: Mapping[str, str], b: Mapping[str, str]) -> Mapping[str, str]:
+    result = {**a, **b}
+    return result
+
+
+def _expand_shell_vars(raw_val: str) -> str:
+    # could be: <string> or ${<env-var-name>} or ${<env-var-name>:-<default-value>}
+    # TODO: implement support for variable substitution and default values
+    # if raw_val is like ${<something>} print a warning and substitute an empty string
+    # otherwise return raw_val
+    match = re.search(r"^\$\{(.*)\}$", raw_val)
+    if match:
+        print(f"WARNING: found unimplemented environment variable substitution: {raw_val}")
+    else:
+        return raw_val
+
+
+# TODO: handle the case where the same env var is defined in multiple places
+def envs_from_compose_file(compose_file_envs: Mapping[str, str]) -> Mapping[str, str]:
+    result = {}
+    for env_var, env_val in compose_file_envs.items():
+        expanded_env_val = _expand_shell_vars(env_val)
+        result.update({env_var: expanded_env_val})
+    return result
+
+
 def envs_from_environment_variables_map(map: Mapping[str, str]) -> List[client.V1EnvVar]:
     result = []
     for env_var, env_val in map.items():
@@ -251,6 +326,12 @@ def generate_kind_config(deployment_dir: Path, deployment_context):
         "apiVersion: kind.x-k8s.io/v1alpha4\n"
         "nodes:\n"
         "- role: control-plane\n"
+        "  kubeadmConfigPatches:\n"
+        "    - |\n"
+        "      kind: InitConfiguration\n"
+        "      nodeRegistration:\n"
+        "        kubeletExtraArgs:\n"
+        "          node-labels: \"ingress-ready=true\"\n"
         f"{port_mappings_yml}\n"
         f"{mounts_yml}\n"
     )

stack_orchestrator/deploy/spec.py

@@ -13,19 +13,83 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from pathlib import Path
 import typing
+import humanfriendly
+
+from pathlib import Path
+
 from stack_orchestrator.util import get_yaml
 from stack_orchestrator import constants
 
 
+class ResourceLimits:
+    cpus: float = None
+    memory: int = None
+    storage: int = None
+
+    def __init__(self, obj=None):
+        if obj is None:
+            obj = {}
+        if "cpus" in obj:
+            self.cpus = float(obj["cpus"])
+        if "memory" in obj:
+            self.memory = humanfriendly.parse_size(obj["memory"])
+        if "storage" in obj:
+            self.storage = humanfriendly.parse_size(obj["storage"])
+
+    def __len__(self):
+        return len(self.__dict__)
+
+    def __iter__(self):
+        for k in self.__dict__:
+            yield k, self.__dict__[k]
+
+    def __repr__(self):
+        return str(self.__dict__)
+
+
+class Resources:
+    limits: ResourceLimits = None
+    reservations: ResourceLimits = None
+
+    def __init__(self, obj=None):
+        if obj is None:
+            obj = {}
+        if "reservations" in obj:
+            self.reservations = ResourceLimits(obj["reservations"])
+        if "limits" in obj:
+            self.limits = ResourceLimits(obj["limits"])
+
+    def __len__(self):
+        return len(self.__dict__)
+
+    def __iter__(self):
+        for k in self.__dict__:
+            yield k, self.__dict__[k]
+
+    def __repr__(self):
+        return str(self.__dict__)
+
+
 class Spec:
 
     obj: typing.Any
     file_path: Path
 
-    def __init__(self) -> None:
-        pass
+    def __init__(self, file_path: Path = None, obj=None) -> None:
+        if obj is None:
+            obj = {}
+        self.file_path = file_path
+        self.obj = obj
+
+    def __getitem__(self, item):
+        return self.obj[item]
+
+    def __contains__(self, item):
+        return item in self.obj
+
+    def get(self, item, default=None):
+        return self.obj.get(item, default)
 
     def init_from_file(self, file_path: Path):
         with file_path:
@@ -33,22 +97,44 @@ class Spec:
             self.file_path = file_path
 
     def get_image_registry(self):
-        return (self.obj[constants.image_resigtry_key]
-                if self.obj and constants.image_resigtry_key in self.obj
-                else None)
+        return self.obj.get(constants.image_registry_key)
 
     def get_volumes(self):
-        return (self.obj["volumes"]
-                if self.obj and "volumes" in self.obj
-                else {})
+        return self.obj.get(constants.volumes_key, {})
 
     def get_configmaps(self):
-        return (self.obj["configmaps"]
-                if self.obj and "configmaps" in self.obj
-                else {})
+        return self.obj.get(constants.configmaps_key, {})
+
+    def get_container_resources(self):
+        return Resources(self.obj.get(constants.resources_key, {}).get("containers", {}))
+
+    def get_volume_resources(self):
+        return Resources(self.obj.get(constants.resources_key, {}).get(constants.volumes_key, {}))
 
     def get_http_proxy(self):
-        return (self.obj[constants.network_key][constants.http_proxy_key]
-                if self.obj and constants.network_key in self.obj
-                and constants.http_proxy_key in self.obj[constants.network_key]
-                else None)
+        return self.obj.get(constants.network_key, {}).get(constants.http_proxy_key, [])
+
+    def get_annotations(self):
+        return self.obj.get(constants.annotations_key, {})
+
+    def get_labels(self):
+        return self.obj.get(constants.labels_key, {})
+
+    def get_privileged(self):
+        return "true" == str(self.obj.get(constants.security_key, {}).get("privileged", "false")).lower()
+
+    def get_capabilities(self):
+        return self.obj.get(constants.security_key, {}).get("capabilities", [])
+
+    def get_deployment_type(self):
+        return self.obj.get(constants.deploy_to_key)
+
+    def is_kubernetes_deployment(self):
+        return self.get_deployment_type() in [constants.k8s_kind_deploy_type,
+                                              constants.k8s_deploy_type]
+
+    def is_kind_deployment(self):
+        return self.get_deployment_type() in [constants.k8s_kind_deploy_type]
+
+    def is_docker_deployment(self):
+        return self.get_deployment_type() in [constants.compose_deploy_type]

stack_orchestrator/deploy/webapp/deploy_webapp.py

@@ -15,6 +15,7 @@
 
 import click
 import os
+import sys
 from pathlib import Path
 from urllib.parse import urlparse
 from tempfile import NamedTemporaryFile
@@ -23,6 +24,7 @@ from stack_orchestrator.util import error_exit, global_options2
 from stack_orchestrator.deploy.deployment_create import init_operation, create_operation
 from stack_orchestrator.deploy.deploy import create_deploy_context
 from stack_orchestrator.deploy.deploy_types import DeployCommandContext
+from stack_orchestrator.deploy.webapp.util import TlsDetails
 
 
 def _fixup_container_tag(deployment_dir: str, image: str):
@@ -36,16 +38,16 @@ def _fixup_container_tag(deployment_dir: str, image: str):
         wfile.write(contents)
 
 
-def _fixup_url_spec(spec_file_name: str, url: str):
+def _fixup_url_spec(spec_file_name: str, url: str, tls_details: TlsDetails = TlsDetails()):
     # url is like: https://example.com/path
     parsed_url = urlparse(url)
-    http_proxy_spec = f'''
-  http-proxy:
+    http_proxy_spec = f'''  http-proxy:
     - host-name: {parsed_url.hostname}
       routes:
         - path: '{parsed_url.path if parsed_url.path else "/"}'
-          proxy-to: webapp:3000
-    '''
+          proxy-to: webapp:80
+{tls_details.to_yaml(indent=6)}
+'''
     spec_file_path = Path(spec_file_name)
     with open(spec_file_path) as rfile:
         contents = rfile.read()
@@ -54,7 +56,8 @@ def _fixup_url_spec(spec_file_name: str, url: str):
         wfile.write(contents)
 
 
-def create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
+def create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file,
+                      tls_details: TlsDetails = None):
     # Do the equivalent of:
     # 1. laconic-so --stack webapp-template deploy --deploy-to k8s init --output webapp-spec.yml
     #   --config (eqivalent of the contents of my-config.env)
@@ -86,7 +89,7 @@ def create_deployment(ctx, deployment_dir, image, url, kube_config, image_regist
         None
     )
     # Add the TLS and DNS spec
-    _fixup_url_spec(spec_file_name, url)
+    _fixup_url_spec(spec_file_name, url, tls_details)
     create_operation(
         deploy_command_context,
         spec_file_name,
@@ -116,8 +119,16 @@ def command(ctx):
 @click.option("--image", help="image to deploy", required=True)
 @click.option("--url", help="url to serve", required=True)
 @click.option("--env-file", help="environment file for webapp")
+@click.option("--tls-host", help="Override TLS hostname (eg, '*.mydomain.com')")
+@click.option("--tls-secret", help="Override TLS secret name")
+@click.option("--tls-issuer", help="TLS issuer to use (default: letsencrypt-prod)")
 @click.pass_context
-def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
+def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file, tls_host, tls_secret, tls_issuer):
     '''create a deployment for the specified webapp container'''
 
-    return create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file)
+    if (tls_secret and not tls_host) or (tls_host and not tls_secret):
+        print("Cannot specify --tls-host without --tls-secret", file=sys.stderr)
+        sys.exit(2)
+
+    return create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file,
+                             TlsDetails(tls_host, tls_secret, tls_issuer))

stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py

@@ -19,15 +19,18 @@ import shlex
 import shutil
 import sys
 import tempfile
+import time
+import uuid
 
 import click
 
+from stack_orchestrator.deploy.images import remote_image_exists, add_tags_to_image
 from stack_orchestrator.deploy.webapp import deploy_webapp
-from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
+from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, TimedLogger, TlsDetails,
                                                    build_container_image, push_container_image,
                                                    file_hash, deploy_to_k8s, publish_deployment,
                                                    hostname_for_deployment_request, generate_hostname_for_app,
-                                                   match_owner)
+                                                   match_owner, skip_by_tag)
 
 
 def process_app_deployment_request(
@@ -39,13 +42,20 @@ def process_app_deployment_request(
     dns_suffix,
     deployment_parent_dir,
     kube_config,
-    image_registry
+    image_registry,
+    force_rebuild,
+    tls_details,
+    logger,
 ):
+    logger.log("BEGIN - process_app_deployment_request")
+
     # 1. look up application
     app = laconic.get_record(app_deployment_request.attributes.application, require=True)
+    logger.log(f"Retrieved app record {app_deployment_request.attributes.application}")
 
     # 2. determine dns
     requested_name = hostname_for_deployment_request(app_deployment_request, laconic)
+    logger.log(f"Determined requested name: {requested_name}")
 
     # HACK
     if "." in requested_name:
@@ -63,7 +73,7 @@ def process_app_deployment_request(
             matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.attributes.request, require=True))
 
         if matched_owner:
-            print("Matched DnsRecord ownership:", matched_owner)
+            logger.log(f"Matched DnsRecord ownership: {matched_owner}")
         else:
             raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" %
                             (dns_record.id, app_deployment_request.id))
@@ -87,7 +97,9 @@ def process_app_deployment_request(
     deployment_record = laconic.get_record(app_deployment_crn)
     deployment_dir = os.path.join(deployment_parent_dir, fqdn)
     deployment_config_file = os.path.join(deployment_dir, "config.env")
+    # TODO: Is there any reason not to simplify the hash input to the app_deployment_crn?
     deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest()
+    app_image_shared_tag = f"laconic-webapp/{app.id}:local"
     #   b. check for deployment directory (create if necessary)
     if not os.path.exists(deployment_dir):
         if deployment_record:
@@ -95,16 +107,33 @@ def process_app_deployment_request(
                             (app_deployment_crn, deployment_dir))
         print("deploy_webapp", deployment_dir)
         deploy_webapp.create_deployment(ctx, deployment_dir, deployment_container_tag,
-                                        f"https://{fqdn}", kube_config, image_registry, env_filename)
+                                        f"https://{fqdn}", kube_config, image_registry, env_filename, tls_details)
     elif env_filename:
         shutil.copyfile(env_filename, deployment_config_file)
 
     needs_k8s_deploy = False
     # 6. build container (if needed)
     if not deployment_record or deployment_record.attributes.application != app.id:
-        build_container_image(app, deployment_container_tag)
-        push_container_image(deployment_dir)
         needs_k8s_deploy = True
+        # check if the image already exists
+        shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag)
+        if shared_tag_exists and not force_rebuild:
+            # simply add our unique tag to the existing image and we are done
+            logger.log(f"Using existing app image {app_image_shared_tag} for {deployment_container_tag}")
+            add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
+            logger.log("Tag complete")
+        else:
+            extra_build_args = []  # TODO: pull from request
+            logger.log(f"Building container image {deployment_container_tag}")
+            build_container_image(app, deployment_container_tag, extra_build_args, logger)
+            logger.log("Build complete")
+            logger.log(f"Pushing container image {deployment_container_tag}")
+            push_container_image(deployment_dir, logger)
+            logger.log("Push complete")
+            # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
+            logger.log(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_tag}")
+            add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
+            logger.log("Tag complete")
 
     # 7. update config (if needed)
     if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config:
@@ -112,12 +141,13 @@ def process_app_deployment_request(
 
     # 8. update k8s deployment
     if needs_k8s_deploy:
-        print("Deploying to k8s")
         deploy_to_k8s(
             deployment_record,
             deployment_dir,
+            logger
         )
 
+    logger.log("Publishing deployment to registry.")
     publish_deployment(
         laconic,
         app,
@@ -126,8 +156,11 @@ def process_app_deployment_request(
         dns_record,
         dns_crn,
         deployment_dir,
-        app_deployment_request
+        app_deployment_request,
+        logger
     )
+    logger.log("Publication complete.")
+    logger.log("END - process_app_deployment_request")
 
 
 def load_known_requests(filename):
@@ -162,10 +195,18 @@ def dump_known_requests(filename, requests, status="SEEN"):
 @click.option("--record-namespace-dns", help="eg, crn://laconic/dns")
 @click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments")
 @click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
+@click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="")
+@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="")
+@click.option("--force-rebuild", help="Rebuild even if the image already exists.", is_flag=True)
+@click.option("--log-dir", help="Output build/deployment logs to directory.", default=None)
+@click.option("--tls-host", help="Override TLS hostname (eg, '*.mydomain.com')")
+@click.option("--tls-secret", help="Override TLS secret name")
+@click.option("--tls-issuer", help="TLS issuer to use (default: letsencrypt-prod)")
 @click.pass_context
-def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir,
+def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir,  # noqa: C901
             request_id, discover, state_file, only_update_state,
-            dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run):
+            dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run,
+            include_tags, exclude_tags, force_rebuild, log_dir, tls_host, tls_secret, tls_issuer):
     if request_id and discover:
         print("Cannot specify both --request-id and --discover", file=sys.stderr)
         sys.exit(2)
@@ -183,6 +224,14 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
             print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr)
             sys.exit(2)
 
+    if (tls_secret and not tls_host) or (tls_host and not tls_secret):
+        print("Cannot specify --tls-host without --tls-secret", file=sys.stderr)
+        sys.exit(2)
+
+    # Split CSV and clean up values.
+    include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
+    exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
+
     laconic = LaconicRegistryClient(laconic_config)
 
     # Find deployment requests.
@@ -204,6 +253,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
     requests.sort(key=lambda r: r.createTime)
     requests.reverse()
     requests_by_name = {}
+    skipped_by_name = {}
     for r in requests:
         # TODO: Do this _after_ filtering deployments and cancellations to minimize round trips.
         app = laconic.get_record(r.attributes.application)
@@ -216,17 +266,20 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
             requested_name = generate_hostname_for_app(app)
             print("Generating name %s for request %s." % (requested_name, r.id))
 
-        if requested_name not in requests_by_name:
-            print(
-                "Found request %s to run application %s on %s."
-                % (r.id, r.attributes.application, requested_name)
-            )
-            requests_by_name[requested_name] = r
-        else:
-            print(
-                "Ignoring request %s, it is superseded by %s."
-                % (r.id, requests_by_name[requested_name].id)
-            )
+        if requested_name in skipped_by_name or requested_name in requests_by_name:
+            print("Ignoring request %s, it has been superseded." % r.id)
+            continue
+
+        if skip_by_tag(r, include_tags, exclude_tags):
+            print("Skipping request %s, filtered by tag (include %s, exclude %s, present %s)" % (r.id,
+                                                                                                 include_tags,
+                                                                                                 exclude_tags,
+                                                                                                 r.attributes.tags))
+            skipped_by_name[requested_name] = r
+            continue
+
+        print("Found request %s to run application %s on %s." % (r.id, r.attributes.application, requested_name))
+        requests_by_name[requested_name] = r
 
     # Find deployments.
     deployments = laconic.app_deployments()
@@ -260,21 +313,45 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
     print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))
 
     if not dry_run:
+        tls_details = TlsDetails(tls_host, tls_secret, tls_issuer)
         for r in requests_to_execute:
             dump_known_requests(state_file, [r], "DEPLOYING")
             status = "ERROR"
+            run_log_file = None
+            run_reg_client = laconic
             try:
+                run_id = f"{r.id}-{str(time.time()).split('.')[0]}-{str(uuid.uuid4()).split('-')[0]}"
+                if log_dir:
+                    run_log_dir = os.path.join(log_dir, r.id)
+                    if not os.path.exists(run_log_dir):
+                        os.mkdir(run_log_dir)
+                    run_log_file_path = os.path.join(run_log_dir, f"{run_id}.log")
+                    print(f"Directing deployment logs to: {run_log_file_path}")
+                    run_log_file = open(run_log_file_path, "wt")
+                    run_reg_client = LaconicRegistryClient(laconic_config, log_file=run_log_file)
+
+                logger = TimedLogger(run_id, run_log_file)
+                logger.log("Processing ...")
                 process_app_deployment_request(
                     ctx,
-                    laconic,
+                    run_reg_client,
                     r,
                     record_namespace_deployments,
                     record_namespace_dns,
                     dns_suffix,
                     os.path.abspath(deployment_parent_dir),
                     kube_config,
-                    image_registry
+                    image_registry,
+                    force_rebuild,
+                    tls_details,
+                    logger
                 )
                 status = "DEPLOYED"
+            except Exception as e:
+                logger.log("ERROR: " + str(e))
             finally:
+                if logger:
+                    logger.log(f"DONE with status {status}", show_step_time=False, show_total_time=True)
                 dump_known_requests(state_file, [r], status)
+                if run_log_file:
+                    run_log_file.close()

stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py

@@ -20,7 +20,7 @@ import sys
 
 import click
 
-from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient, match_owner
+from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient, match_owner, skip_by_tag
 
 
 def process_app_removal_request(ctx,
@@ -107,10 +107,12 @@ def dump_known_requests(filename, requests):
 @click.option("--delete-names/--preserve-names", help="Delete all names associated with removed deployments.", default=True)
 @click.option("--delete-volumes/--preserve-volumes", default=True, help="delete data volumes")
 @click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
+@click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="")
+@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="")
 @click.pass_context
 def command(ctx, laconic_config, deployment_parent_dir,
             request_id, discover, state_file, only_update_state,
-            delete_names, delete_volumes, dry_run):
+            delete_names, delete_volumes, dry_run, include_tags, exclude_tags):
     if request_id and discover:
         print("Cannot specify both --request-id and --discover", file=sys.stderr)
         sys.exit(2)
@@ -123,6 +125,10 @@ def command(ctx, laconic_config, deployment_parent_dir,
         print("--only-update-state requires --state-file", file=sys.stderr)
         sys.exit(2)
 
+    # Split CSV and clean up values.
+    include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
+    exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
+
     laconic = LaconicRegistryClient(laconic_config)
 
     # Find deployment removal requests.
@@ -141,6 +147,7 @@ def command(ctx, laconic_config, deployment_parent_dir,
 
     previous_requests = load_known_requests(state_file)
     requests.sort(key=lambda r: r.createTime)
+    requests.reverse()
 
     # Find deployments.
     deployments = {}
@@ -155,10 +162,22 @@ def command(ctx, laconic_config, deployment_parent_dir,
             # TODO: should we handle CRNs?
             removals_by_deployment[r.attributes.deployment] = r
 
-    requests_to_execute = []
+    one_per_deployment = {}
     for r in requests:
         if not r.attributes.deployment:
             print(f"Skipping removal request {r.id} since it was a cancellation.")
+        elif r.attributes.deployment in one_per_deployment:
+            print(f"Skipping removal request {r.id} since it was superseded.")
+        else:
+            one_per_deployment[r.attributes.deployment] = r
+
+    requests_to_execute = []
+    for r in one_per_deployment.values():
+        if skip_by_tag(r, include_tags, exclude_tags):
+            print("Skipping removal request %s, filtered by tag (include %s, exclude %s, present %s)" % (r.id,
+                                                                                                         include_tags,
+                                                                                                         exclude_tags,
+                                                                                                         r.attributes.tags))
         elif r.id in removals_by_request:
             print(f"Found satisfied request for {r.id} at {removals_by_request[r.id].id}")
         elif r.attributes.deployment in removals_by_deployment:

stack_orchestrator/deploy/webapp/util.py

@@ -13,6 +13,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
+import datetime
 import hashlib
 import json
 import os
@@ -25,6 +26,40 @@ import uuid
 import yaml
 
 
+class TlsDetails:
+    def __init__(self, host_or_hosts=None, secret_name: str = None, issuer_name: str = None):
+        if host_or_hosts:
+            if isinstance(host_or_hosts, list):
+                self.hosts = host_or_hosts
+            else:
+                self.hosts = [host_or_hosts]
+        else:
+            self.hosts = None
+        self.secret_name = secret_name
+        self.issuer_name = issuer_name
+
+    def to_yaml(self, indent=6):
+        if not self.hosts and not self.secret_name and not self.issuer_name:
+            return ""
+
+        ret = " " * indent + "tls:\n"
+        indent += 2
+
+        if self.issuer_name:
+            ret += " " * indent + "issuer: '%s'\n" % self.issuer_name
+
+        if self.secret_name:
+            ret += " " * indent + "secret: '%s'\n" % self.secret_name
+
+        if self.hosts:
+            ret += " " * indent + "hosts:"
+            indent += 2
+            for h in self.hosts:
+                ret += "\n" + " " * indent + "- '%s'" % h
+
+        return ret
+
+
 class AttrDict(dict):
     def __init__(self, *args, **kwargs):
         super(AttrDict, self).__init__(*args, **kwargs)
@@ -39,13 +74,38 @@ class AttrDict(dict):
             return v
 
 
-def cmd(*vargs):
+class TimedLogger:
+    def __init__(self, id="", file=None):
+        self.start = datetime.datetime.now()
+        self.last = self.start
+        self.id = id
+        self.file = file
+
+    def log(self, msg, show_step_time=True, show_total_time=False):
+        prefix = f"{datetime.datetime.utcnow()} - {self.id}"
+        if show_step_time:
+            prefix += f" - {datetime.datetime.now() - self.last} (step)"
+        if show_total_time:
+            prefix += f" - {datetime.datetime.now() - self.start} (total)"
+        print(f"{prefix}: {msg}", file=self.file)
+        if self.file:
+            self.file.flush()
+        self.last = datetime.datetime.now()
+
+
+def logged_cmd(log_file, *vargs):
+    result = None
     try:
+        if log_file:
+            print(" ".join(vargs), file=log_file)
         result = subprocess.run(vargs, capture_output=True)
         result.check_returncode()
         return result.stdout.decode()
     except Exception as err:
-        print(result.stderr.decode())
+        if result:
+            print(result.stderr.decode(), file=log_file)
+        else:
+            print(str(err), file=log_file)
         raise err
 
 
@@ -58,8 +118,9 @@ def match_owner(recordA, *records):
 
 
 class LaconicRegistryClient:
-    def __init__(self, config_file):
+    def __init__(self, config_file, log_file=None):
         self.config_file = config_file
+        self.log_file = log_file
         self.cache = AttrDict(
             {
                 "name_or_id": {},
@@ -77,7 +138,7 @@ class LaconicRegistryClient:
                 args.append("--%s" % k)
                 args.append(str(v))
 
-        results = [AttrDict(r) for r in json.loads(cmd(*args))]
+        results = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args))]
 
         # Most recent records first
         results.sort(key=lambda r: r.createTime)
@@ -115,7 +176,7 @@ class LaconicRegistryClient:
 
         args = ["laconic", "-c", self.config_file, "cns", "name", "resolve", name]
 
-        parsed = [AttrDict(r) for r in json.loads(cmd(*args))]
+        parsed = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args))]
         if parsed:
             self._add_to_cache(parsed)
             return parsed[0]
@@ -145,7 +206,7 @@ class LaconicRegistryClient:
             name_or_id,
         ]
 
-        parsed = [AttrDict(r) for r in json.loads(cmd(*args))]
+        parsed = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args))]
         if len(parsed):
             self._add_to_cache(parsed)
             return parsed[0]
@@ -173,22 +234,22 @@ class LaconicRegistryClient:
             record_file = open(record_fname, 'w')
             yaml.dump(record, record_file)
             record_file.close()
-            print(open(record_fname, 'r').read())
+            print(open(record_fname, 'r').read(), file=self.log_file)
 
             new_record_id = json.loads(
-                cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname)
+                logged_cmd(self.log_file, "laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname)
             )["id"]
             for name in names:
                 self.set_name(name, new_record_id)
             return new_record_id
         finally:
-            cmd("rm", "-rf", tmpdir)
+            logged_cmd(self.log_file, "rm", "-rf", tmpdir)
 
     def set_name(self, name, record_id):
-        cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, record_id)
+        logged_cmd(self.log_file, "laconic", "-c", self.config_file, "cns", "name", "set", name, record_id)
 
     def delete_name(self, name):
-        cmd("laconic", "-c", self.config_file, "cns", "name", "delete", name)
+        logged_cmd(self.log_file, "laconic", "-c", self.config_file, "cns", "name", "delete", name)
 
 
 def file_hash(filename):
@@ -212,7 +273,7 @@ def determine_base_container(clone_dir, app_type="webapp"):
     return base_container
 
 
-def build_container_image(app_record, tag, extra_build_args=[]):
+def build_container_image(app_record, tag, extra_build_args=[], logger=None):
     tmpdir = tempfile.mkdtemp()
 
     try:
@@ -221,23 +282,33 @@ def build_container_image(app_record, tag, extra_build_args=[]):
         repo = random.choice(app_record.attributes.repository)
         clone_dir = os.path.join(tmpdir, record_id)
 
-        print(f"Cloning repository {repo} to {clone_dir} ...")
+        logger.log(f"Cloning repository {repo} to {clone_dir} ...")
         if ref:
             # TODO: Determing branch or hash, and use depth 1 if we can.
             git_env = dict(os.environ.copy())
             # Never prompt
             git_env["GIT_TERMINAL_PROMPT"] = "0"
-            subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env)
-            subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env)
+            try:
+                subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env, stdout=logger.file, stderr=logger.file)
+            except Exception as e:
+                logger.log(f"git clone failed.  Is the repository {repo} private?")
+                raise e
+            try:
+                subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env, stdout=logger.file, stderr=logger.file)
+            except Exception as e:
+                logger.log(f"git checkout failed.  Does ref {ref} exist?")
+                raise e
         else:
-            result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir])
+            result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir], stdout=logger.file, stderr=logger.file)
             result.check_returncode()
 
         base_container = determine_base_container(clone_dir, app_record.attributes.app_type)
 
-        print("Building webapp ...")
+        logger.log("Building webapp ...")
         build_command = [
-            sys.argv[0], "build-webapp",
+            sys.argv[0],
+            "--verbose",
+            "build-webapp",
             "--source-repo", clone_dir,
             "--tag", tag,
             "--base-container", base_container
@@ -246,26 +317,31 @@ def build_container_image(app_record, tag, extra_build_args=[]):
             build_command.append("--extra-build-args")
             build_command.append(" ".join(extra_build_args))
 
-        result = subprocess.run(build_command)
+        result = subprocess.run(build_command, stdout=logger.file, stderr=logger.file)
         result.check_returncode()
     finally:
-        cmd("rm", "-rf", tmpdir)
+        logged_cmd(logger.file, "rm", "-rf", tmpdir)
 
 
-def push_container_image(deployment_dir):
-    print("Pushing image ...")
-    result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"])
+def push_container_image(deployment_dir, logger):
+    logger.log("Pushing images ...")
+    result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"],
+                            stdout=logger.file, stderr=logger.file)
     result.check_returncode()
+    logger.log("Finished pushing images.")
 
 
-def deploy_to_k8s(deploy_record, deployment_dir):
+def deploy_to_k8s(deploy_record, deployment_dir, logger):
     if not deploy_record:
         command = "up"
     else:
         command = "update"
 
-    result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command])
+    logger.log("Deploying to k8s ...")
+    result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command],
+                            stdout=logger.file, stderr=logger.file)
     result.check_returncode()
+    logger.log("Finished deploying to k8s.")
 
 
 def publish_deployment(laconic: LaconicRegistryClient,
@@ -275,7 +351,8 @@ def publish_deployment(laconic: LaconicRegistryClient,
                        dns_record,
                        dns_crn,
                        deployment_dir,
-                       app_deployment_request=None):
+                       app_deployment_request=None,
+                       logger=None):
     if not deploy_record:
         deploy_ver = "0.0.1"
     else:
@@ -305,6 +382,8 @@ def publish_deployment(laconic: LaconicRegistryClient,
     if app_deployment_request:
         new_dns_record["record"]["request"] = app_deployment_request.id
 
+    if logger:
+        logger.log("Publishing DnsRecord.")
     dns_id = laconic.publish(new_dns_record, [dns_crn])
 
     new_deployment_record = {
@@ -324,6 +403,8 @@ def publish_deployment(laconic: LaconicRegistryClient,
     if app_deployment_request:
         new_deployment_record["record"]["request"] = app_deployment_request.id
 
+    if logger:
+        logger.log("Publishing ApplicationDeploymentRecord.")
     deployment_id = laconic.publish(new_deployment_record, [deployment_crn])
     return {"dns": dns_id, "deployment": deployment_id}
 
@@ -349,3 +430,17 @@ def generate_hostname_for_app(app):
     else:
         m.update(app.attributes.repository.encode())
     return "%s-%s" % (last_part, m.hexdigest()[0:10])
+
+
+def skip_by_tag(r, include_tags, exclude_tags):
+    for tag in exclude_tags:
+        if r.attributes.tags and tag in r.attributes.tags:
+            return True
+
+    if include_tags:
+        for tag in include_tags:
+            if r.attributes.tags and tag in r.attributes.tags:
+                return False
+        return True
+
+    return False

stack_orchestrator/main.py

@@ -17,7 +17,7 @@ import click
 
 from stack_orchestrator.command_types import CommandOptions
 from stack_orchestrator.repos import setup_repositories
-from stack_orchestrator.build import build_containers
+from stack_orchestrator.build import build_containers, fetch_containers
 from stack_orchestrator.build import build_npms
 from stack_orchestrator.build import build_webapp
 from stack_orchestrator.deploy.webapp import (run_webapp,
@@ -52,6 +52,7 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err
 
 cli.add_command(setup_repositories.command, "setup-repositories")
 cli.add_command(build_containers.command, "build-containers")
+cli.add_command(fetch_containers.command, "fetch-containers")
 cli.add_command(build_npms.command, "build-npms")
 cli.add_command(build_webapp.command, "build-webapp")
 cli.add_command(run_webapp.command, "run-webapp")

stack_orchestrator/repos/setup_repositories.py

@@ -26,7 +26,7 @@ import importlib.resources
 from pathlib import Path
 import yaml
 from stack_orchestrator.constants import stack_file_name
-from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
+from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit, warn_exit
 
 
 class GitProgress(git.RemoteProgress):
@@ -249,8 +249,8 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches
             error_exit(f"stack {stack} does not exist")
         with stack_file_path:
             stack_config = yaml.safe_load(open(stack_file_path, "r"))
-            if "repos" not in stack_config:
-                error_exit(f"stack {stack} does not define any repositories")
+            if "repos" not in stack_config or stack_config["repos"] is None:
+                warn_exit(f"stack {stack} does not define any repositories")
             else:
                 repos_in_scope = stack_config["repos"]
     else:

stack_orchestrator/util.py

@@ -146,6 +146,12 @@ def get_config_file_dir():
     return source_config_dir
 
 
+def get_k8s_dir():
+    data_dir = Path(__file__).absolute().parent.joinpath("data")
+    source_config_dir = data_dir.joinpath("k8s")
+    return source_config_dir
+
+
 def get_parsed_deployment_spec(spec_file):
     spec_file_path = Path(spec_file)
     try:
@@ -189,5 +195,10 @@ def error_exit(s):
     sys.exit(1)
 
 
+def warn_exit(s):
+    print(f"WARN: {s}")
+    sys.exit(0)
+
+
 def env_var_map_from_file(file: Path) -> Mapping[str, str]:
     return dotenv_values(file)

tests/container-registry/run-test.sh

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+    # Dump environment variables for debugging
+    echo "Environment variables:"
+    env
+fi
+
+stack="container-registry"
+
+# Helper functions: TODO move into a separate file
+wait_for_pods_started () {
+    for i in {1..50}
+    do
+        local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps )
+
+        if [[ "$ps_output" == *"Running containers:"* ]]; then
+            # if ready, return
+            return
+        else
+            # if not ready, wait
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "waiting for pods to start: FAILED"
+    delete_cluster_exit
+}
+
+wait_for_log_output () {
+    for i in {1..50}
+    do
+
+        local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+
+        if [[ ! -z "$log_output" ]]; then
+            # if ready, return
+            return
+        else
+            # if not ready, wait
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "waiting for pods log content: FAILED"
+    delete_cluster_exit
+}
+
+
+delete_cluster_exit () {
+    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+    exit 1
+}
+
+# Note: eventually this test should be folded into ../deploy/
+# but keeping it separate for now for convenience
+TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+# Set a non-default repo dir
+export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
+echo "Testing this package: $TEST_TARGET_SO"
+echo "Test version command"
+reported_version_string=$( $TEST_TARGET_SO version )
+echo "Version reported is: ${reported_version_string}"
+echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
+rm -rf $CERC_REPO_BASE_DIR
+mkdir -p $CERC_REPO_BASE_DIR
+$TEST_TARGET_SO --stack ${stack} setup-repositories
+$TEST_TARGET_SO --stack ${stack} build-containers
+# Test basic stack-orchestrator deploy to k8s
+test_deployment_dir=$CERC_REPO_BASE_DIR/${stack}-deployment-dir
+test_deployment_spec=$CERC_REPO_BASE_DIR/${stack}-deployment-spec.yml
+$TEST_TARGET_SO --stack ${stack} deploy --deploy-to k8s-kind init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED
+# Check the file now exists
+if [ ! -f "$test_deployment_spec" ]; then
+    echo "deploy init test: spec file not present"
+    echo "deploy init test: FAILED"
+    exit 1
+fi
+echo "deploy init test: passed"
+
+# Switch to a full path for bind mount.
+volume_name="registry-data"
+sed -i "s|^\(\s*${volume_name}:$\)$|\1 ${test_deployment_dir}/data/${volume_name}|" $test_deployment_spec
+
+# Add ingress config to the spec file
+ed $test_deployment_spec <<IngressSpec
+/network:/
+a
+  http-proxy:
+    - host-name: localhost
+      routes:
+        - path: /
+          proxy-to: registry:5000
+.
+w
+q
+IngressSpec
+
+$TEST_TARGET_SO --stack ${stack} deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
+# Check the deployment dir exists
+if [ ! -d "$test_deployment_dir" ]; then
+    echo "deploy create test: deployment directory not present"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+echo "deploy create test: passed"
+
+# Note: this isn't strictly necessary, except we end up trying to push the image into
+# the kind cluster then fails because it can't be found locally
+docker pull registry:2.8
+
+# Try to start the deployment
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+wait_for_pods_started
+# Check logs command works
+wait_for_log_output
+sleep 1
+log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_3" == *"listening on"* ]]; then
+    echo "deployment logs test: passed"
+else
+    echo "deployment logs test: FAILED"
+    echo $log_output_3
+    delete_cluster_exit
+fi
+
+# Check that we can use the registry
+# Note: since this pulls from the DockerCo registry without auth it's possible it'll run into rate limiting issues
+docker pull hello-world
+docker tag hello-world localhost:80/hello-world
+docker push localhost:80/hello-world
+# Then do a quick check that we actually pushed something there
+# See: https://stackoverflow.com/questions/31251356/how-to-get-a-list-of-images-on-docker-registry-v2
+registry_response=$(curl -s -X GET http://localhost:80/v2/_catalog)
+if [[ "$registry_response" == *"{\"repositories\":[\"hello-world\"]}"* ]]; then
+    echo "registry content test: passed"
+else
+    echo "registry content test: FAILED"
+    echo $registry_response
+    delete_cluster_exit
+fi
+
+# Stop and clean up
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+echo "Test passed"

tests/database/run-test.sh

@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+    # Dump environment variables for debugging
+    echo "Environment variables:"
+    env
+fi
+
+if [ "$1" == "from-path" ]; then
+    TEST_TARGET_SO="laconic-so"
+else
+    TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+fi
+
+stack="test-database"
+spec_file=${stack}-spec.yml
+deployment_dir=${stack}-deployment
+
+# Helper functions: TODO move into a separate file
+wait_for_pods_started () {
+    for i in {1..50}
+    do
+        local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps )
+
+        if [[ "$ps_output" == *"Running containers:"* ]]; then
+            # if ready, return
+            return
+        else
+            # if not ready, wait
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "waiting for pods to start: FAILED"
+    delete_cluster_exit
+}
+
+wait_for_test_complete () {
+    for i in {1..50}
+    do
+
+        local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+
+    if [[ "${log_output}" == *"Database test client: test complete"* ]]; then
+            # if ready, return
+            return
+        else
+            # if not ready, wait
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "waiting for test complete: FAILED"
+    delete_cluster_exit
+}
+
+
+delete_cluster_exit () {
+    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+    exit 1
+}
+
+# Set a non-default repo dir
+export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
+echo "Testing this package: $TEST_TARGET_SO"
+echo "Test version command"
+reported_version_string=$( $TEST_TARGET_SO version )
+echo "Version reported is: ${reported_version_string}"
+echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
+rm -rf $CERC_REPO_BASE_DIR
+mkdir -p $CERC_REPO_BASE_DIR
+$TEST_TARGET_SO --stack ${stack} setup-repositories
+$TEST_TARGET_SO --stack ${stack} build-containers
+# Test basic stack-orchestrator deploy to k8s
+test_deployment_dir=$CERC_REPO_BASE_DIR/${deployment_dir}
+test_deployment_spec=$CERC_REPO_BASE_DIR/${spec_file}
+
+$TEST_TARGET_SO --stack ${stack} deploy --deploy-to k8s-kind init --output $test_deployment_spec
+# Check the file now exists
+if [ ! -f "$test_deployment_spec" ]; then
+    echo "deploy init test: spec file not present"
+    echo "deploy init test: FAILED"
+    exit 1
+fi
+echo "deploy init test: passed"
+
+# Switch to a full path for the data dir so it gets provisioned as a host bind mounted volume and preserved beyond cluster lifetime
+sed -i "s|^\(\s*db-data:$\)$|\1 ${test_deployment_dir}/data/db-data|" $test_deployment_spec
+
+$TEST_TARGET_SO --stack ${stack} deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
+# Check the deployment dir exists
+if [ ! -d "$test_deployment_dir" ]; then
+    echo "deploy create test: deployment directory not present"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+echo "deploy create test: passed"
+
+# Try to start the deployment
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+wait_for_pods_started
+# Check logs command works
+wait_for_test_complete
+log_output_1=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_1" == *"Database test client: test data does not exist"* ]]; then
+    echo "Create database content test: passed"
+else
+    echo "Create database content test: FAILED"
+    delete_cluster_exit
+fi
+
+# Stop then start again and check the volume was preserved
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
+# Sleep a bit just in case
+sleep 20
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+wait_for_pods_started
+wait_for_test_complete
+
+log_output_2=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_2" == *"Database test client: test data already exists"* ]]; then
+    echo "Retain database content test: passed"
+else
+    echo "Retain database content test: FAILED"
+    delete_cluster_exit
+fi
+
+# Stop and clean up
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+echo "Test passed"

tests/deploy/run-deploy-test.sh

@@ -6,6 +6,12 @@ fi
 # Dump environment variables for debugging
 echo "Environment variables:"
 env
+
+delete_cluster_exit () {
+    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+    exit 1
+}
+
 # Test basic stack-orchestrator deploy
 echo "Running stack-orchestrator deploy test"
 # Bit of a hack, test the most recent package
@@ -57,7 +63,7 @@ $TEST_TARGET_SO --stack test deploy down
 # The next time we bring the container up the volume will be old (from the previous run above)
 $TEST_TARGET_SO --stack test deploy up
 log_output_1=$( $TEST_TARGET_SO --stack test deploy logs )
-if [[ "$log_output_1" == *"Filesystem is old"* ]]; then
+if [[ "$log_output_1" == *"filesystem is old"* ]]; then
     echo "Retain volumes test: passed"
 else
     echo "Retain volumes test: FAILED"
@@ -67,7 +73,7 @@ $TEST_TARGET_SO --stack test deploy down --delete-volumes
 # Now when we bring the container up the volume will be new again
 $TEST_TARGET_SO --stack test deploy up
 log_output_2=$( $TEST_TARGET_SO --stack test deploy logs )
-if [[ "$log_output_2" == *"Filesystem is fresh"* ]]; then
+if [[ "$log_output_2" == *"filesystem is fresh"* ]]; then
     echo "Delete volumes test: passed"
 else
     echo "Delete volumes test: FAILED"
@@ -77,7 +83,7 @@ $TEST_TARGET_SO --stack test deploy down --delete-volumes
 # Basic test of creating a deployment
 test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
 test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml
-$TEST_TARGET_SO --stack test deploy init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED
+$TEST_TARGET_SO --stack test deploy init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED,CERC_TEST_PARAM_3=FAST
 # Check the file now exists
 if [ ! -f "$test_deployment_spec" ]; then
     echo "deploy init test: spec file not present"
@@ -106,12 +112,16 @@ if [ ! "$create_file_content" == "create-command-output-data"  ]; then
     echo "deploy create test: FAILED"
     exit 1
 fi
+
+# Add a config file to be picked up by the ConfigMap before starting.
+echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/data/test-config/test_config
+
 echo "deploy create output file test: passed"
 # Try to start the deployment
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
 # Check logs command works
 log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then
+if [[ "$log_output_3" == *"filesystem is fresh"* ]]; then
     echo "deployment logs test: passed"
 else
     echo "deployment logs test: FAILED"
@@ -124,6 +134,44 @@ else
     echo "deployment config test: FAILED"
     exit 1
 fi
+# Check the config variable CERC_TEST_PARAM_2 was passed correctly from the compose file
+if [[ "$log_output_3" == *"Test-param-2: CERC_TEST_PARAM_2_VALUE"* ]]; then
+    echo "deployment compose config test: passed"
+else
+    echo "deployment compose config test: FAILED"
+    exit 1
+fi
+# Check the config variable CERC_TEST_PARAM_3 was passed correctly
+if [[ "$log_output_3" == *"Test-param-3: FAST"* ]]; then
+    echo "deployment config test: passed"
+else
+    echo "deployment config test: FAILED"
+    exit 1
+fi
+
+# Check that the ConfigMap is mounted and contains the expected content.
+log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then
+    echo "deployment ConfigMap test: passed"
+else
+    echo "deployment ConfigMap test: FAILED"
+    delete_cluster_exit
+fi
+
+# Stop then start again and check the volume was preserved
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
+# Sleep a bit just in case
+# sleep for longer to check if that's why the subsequent create cluster fails
+sleep 20
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_5" == *"filesystem is old"* ]]; then
+    echo "Retain volumes test: passed"
+else
+    echo "Retain volumes test: FAILED"
+    delete_cluster_exit
+fi
+
 # Stop and clean up
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
 echo "Test passed"

tests/k8s-deploy/run-deploy-test.sh

@@ -76,6 +76,10 @@ if [ ! -f "$test_deployment_spec" ]; then
     exit 1
 fi
 echo "deploy init test: passed"
+
+# Switch to a full path for bind mount.
+sed -i "s|^\(\s*test-data-bind:$\)$|\1 ${test_deployment_dir}/data/test-data-bind|" $test_deployment_spec
+
 $TEST_TARGET_SO --stack test deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
 # Check the deployment dir exists
 if [ ! -d "$test_deployment_dir" ]; then
@@ -99,7 +103,7 @@ if [ ! "$create_file_content" == "create-command-output-data"  ]; then
 fi
 
 # Add a config file to be picked up by the ConfigMap before starting.
-echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/data/test-config/test_config
+echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/configmaps/test-config/test_config
 
 echo "deploy create output file test: passed"
 # Try to start the deployment
@@ -107,13 +111,16 @@ $TEST_TARGET_SO deployment --dir $test_deployment_dir start
 wait_for_pods_started
 # Check logs command works
 wait_for_log_output
+sleep 1
 log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then
+if [[ "$log_output_3" == *"filesystem is fresh"* ]]; then
     echo "deployment logs test: passed"
 else
     echo "deployment logs test: FAILED"
+    echo $log_output_3
     delete_cluster_exit
 fi
+
 # Check the config variable CERC_TEST_PARAM_1 was passed correctly
 if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then
     echo "deployment config test: passed"
@@ -122,6 +129,14 @@ else
     delete_cluster_exit
 fi
 
+# Check the config variable CERC_TEST_PARAM_2 was passed correctly from the compose file
+if [[ "$log_output_3" == *"Test-param-2: CERC_TEST_PARAM_2_VALUE"* ]]; then
+    echo "deployment compose config test: passed"
+else
+    echo "deployment compose config test: FAILED"
+    exit 1
+fi
+
 # Check that the ConfigMap is mounted and contains the expected content.
 log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
 if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then
@@ -131,6 +146,26 @@ else
     delete_cluster_exit
 fi
 
+# Check that the bind-mount volume is mounted.
+log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_5" == *"/data: MOUNTED"* ]]; then
+    echo "deployment bind volumes test: passed"
+else
+    echo "deployment bind volumes test: FAILED"
+    echo $log_output_5
+    delete_cluster_exit
+fi
+
+# Check that the provisioner managed volume is mounted.
+log_output_6=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_6" == *"/data2: MOUNTED"* ]]; then
+    echo "deployment provisioner volumes test: passed"
+else
+    echo "deployment provisioner volumes test: FAILED"
+    echo $log_output_6
+    delete_cluster_exit
+fi
+
 # Stop then start again and check the volume was preserved
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop
 # Sleep a bit just in case
@@ -139,13 +174,26 @@ sleep 20
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
 wait_for_pods_started
 wait_for_log_output
-log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_5" == *"Filesystem is old"* ]]; then
-    echo "Retain volumes test: passed"
+sleep 1
+
+log_output_10=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_10" == *"/data filesystem is old"* ]]; then
+    echo "Retain bind volumes test: passed"
 else
-    echo "Retain volumes test: FAILED"
+    echo "Retain bind volumes test: FAILED"
     delete_cluster_exit
 fi
+
+# These volumes will be completely destroyed by the kind delete/create, because they lived inside
+# the kind container.  So, unlike the bind-mount case, they will appear fresh after the restart.
+log_output_11=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_11" == *"/data2 filesystem is fresh"* ]]; then
+    echo "Fresh provisioner volumes test: passed"
+else
+    echo "Fresh provisioner volumes test: FAILED"
+    delete_cluster_exit
+fi
+
 # Stop and clean up
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
 echo "Test passed"