Merge branch 'main' into ci-test

Reviewed-on: cerc-io/stack-orchestrator#758
Co-authored-by: David Boreham <david@bozemanpass.com>
Co-committed-by: David Boreham <david@bozemanpass.com>

.gitea/workflows/fixturenet-eth-plugeth-test.yml

@@ -6,6 +6,8 @@ on:
     paths:
       - '!**'
       - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '2 14 * * *'
 
 # Needed until we can incorporate docker startup into the executor container
 env:

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -6,11 +6,8 @@ on:
     paths:
       - '!**'
       - '.gitea/workflows/triggers/fixturenet-laconicd-test'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
+  schedule:
+    - cron: '1 13 * * *'
 
 jobs:
   test:
@@ -47,9 +44,5 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run fixturenet-laconicd tests"
         run: ./tests/fixturenet-laconicd/run-test.sh

.gitea/workflows/lint.yml

@@ -0,0 +1,21 @@
+name: Lint Checks
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches: '*'
+
+jobs:
+  test:
+    name: "Run linter"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name : "Run flake8"
+        uses: py-actions/flake8@v2

.gitea/workflows/test-container-registry.yml

@@ -0,0 +1,54 @@
+name: Container Registry Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-container-registry'
+      - '.gitea/workflows/test-container-registry.yml'
+      - 'tests/container-registry/run-test.sh'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '6 19 * * *'
+
+jobs:
+  test:
+    name: "Run contaier registry hosting test on kind/k8s"
+    runs-on: ubuntu-22.04
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Install ed" # Only needed until we remove the need to edit the spec file
+        run: apt update && apt install -y ed
+      - name: "Run container registry deployment test"
+        run: |
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/container-registry/run-test.sh
+

.gitea/workflows/test-database.yml

@@ -0,0 +1,52 @@
+name: Database Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-database'
+      - '.gitea/workflows/test-database.yml'
+      - 'tests/database/run-test.sh'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '5 18 * * *'
+
+jobs:
+  test:
+    name: "Run database hosting test on kind/k8s"
+    runs-on: ubuntu-22.04
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Run database deployment test"
+        run: |
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          CERC_SCRIPT_DEBUG=1 ./tests/database/run-test.sh
+

.gitea/workflows/test-k8s-deploy.yml

@@ -4,20 +4,19 @@ on:
   pull_request:
     branches: '*'
   push:
-    branches:
-      - main
-      - ci-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-k8s-deploy'
+      - '.gitea/workflows/test-k8s-deploy.yml'
+      - 'tests/k8s-deploy/run-deploy-test.sh'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '3 15 * * *'
 
 jobs:
   test:
-    name: "Run deploy test suite"
-    runs-on: ubuntu-latest
+    name: "Run deploy test suite on kind/k8s"
+    runs-on: ubuntu-22.04
     steps:
       - name: "Clone project repository"
         uses: actions/checkout@v3
@@ -41,15 +40,15 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Run k8s deployment test"
         run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
-      - name: "Install Go"
-        uses: actions/setup-go@v4
-        with:
-          go-version: '1.21'
-      - name: "Install Kind"
-        run: go install sigs.k8s.io/kind@v0.20.0
-      - name: "Debug Kind"
-        run: kind create cluster --retain && docker logs kind-control-plane
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/k8s-deploy/run-deploy-test.sh
+

.gitea/workflows/test-webapp.yml

@@ -41,6 +41,8 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
+      - name: "Install wget" # 20240109 - Only needed until the executors are updated.
+        run: apt update && apt install -y wget
       - name: Start dockerd # Also needed until we can incorporate into the executor
         run: |
           dockerd -H $DOCKER_HOST --userland-proxy=false &

.gitea/workflows/triggers/fixturenet-eth-plugeth-test

@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-eth-plugeth-test CI job
 trigger
+trigger

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-laconicd-test CI job
-
+Trigger
+Trigger

.gitea/workflows/triggers/test-container-registry

@@ -0,0 +1 @@
+Change this file to trigger running the test-container-registry CI job

.gitea/workflows/triggers/test-database

@@ -0,0 +1,2 @@
+Change this file to trigger running the test-database CI job
+Trigger test run

.gitea/workflows/triggers/test-k8s-deploy

@@ -0,0 +1,2 @@
+Change this file to trigger running the test-k8s-deploy CI job
+Trigger test on PR branch

README.md

@@ -29,10 +29,10 @@ chmod +x ~/.docker/cli-plugins/docker-compose
 Next decide on a directory where you would like to put the stack-orchestrator program. Typically this would be 
 a "user" binary directory such as `~/bin` or perhaps `/usr/local/laconic` or possibly just the current working directory.
 
-Now, having selected that directory, download the latest release from [this page](https://github.com/cerc-io/stack-orchestrator/tags) into it (we're using `~/bin` below for concreteness but edit to suit if you selected a different directory). Also be sure that the destination directory exists and is writable:
+Now, having selected that directory, download the latest release from [this page](https://git.vdb.to/cerc-io/stack-orchestrator/tags) into it (we're using `~/bin` below for concreteness but edit to suit if you selected a different directory). Also be sure that the destination directory exists and is writable:
 
 ```bash
-curl -L -o ~/bin/laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+curl -L -o ~/bin/laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
 ```
 
 Give it execute permissions:
@@ -52,7 +52,7 @@ Version: 1.1.0-7a607c2-202304260513
 Save the distribution url to `~/.laconic-so/config.yml`:
 ```bash
 mkdir ~/.laconic-so
-echo "distribution-url: https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so" >  ~/.laconic-so/config.yml
+echo "distribution-url: https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so" >  ~/.laconic-so/config.yml
 ```
 
 ### Update

docs/CONTRIBUTING.md

@@ -26,7 +26,7 @@ In addition to the pre-requisites listed in the [README](/README.md), the follow
 
 1. Clone this repository:
    ```
-   $ git clone https://github.com/cerc-io/stack-orchestrator.git
+   $ git clone https://git.vdb.to/cerc-io/stack-orchestrator.git
    ```
 
 2. Enter the project directory:

docs/adding-a-new-stack.md

@@ -1,10 +1,10 @@
 # Adding a new stack
 
-See [this PR](https://github.com/cerc-io/stack-orchestrator/pull/434) for an example of how to currently add a minimal stack to stack orchestrator. The [reth stack](https://github.com/cerc-io/stack-orchestrator/pull/435) is another good example.
+See [this PR](https://git.vdb.to/cerc-io/stack-orchestrator/pull/434) for an example of how to currently add a minimal stack to stack orchestrator. The [reth stack](https://git.vdb.to/cerc-io/stack-orchestrator/pull/435) is another good example.
 
 For external developers, we recommend forking this repo and adding your stack directly to your fork. This initially requires running in "developer mode" as described [here](/docs/CONTRIBUTING.md). Check out the [Namada stack](https://github.com/vknowable/stack-orchestrator/blob/main/app/data/stacks/public-namada/digitalocean_quickstart.md) from Knowable to see how that is done.
 
-Core to the feature completeness of stack orchestrator is to [decouple the tool functionality from payload](https://github.com/cerc-io/stack-orchestrator/issues/315) which will no longer require forking to add a stack.
+Core to the feature completeness of stack orchestrator is to [decouple the tool functionality from payload](https://git.vdb.to/cerc-io/stack-orchestrator/issues/315) which will no longer require forking to add a stack.
 
 ## Example
 

docs/laconicd-with-console.md

@@ -15,132 +15,139 @@ To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the
 16 GB Memory / 8 Intel vCPUs / 160 GB Disk.
 
 1. Login to the droplet as root (either by SSH key or password set in the DO console)
+    ```
+    ssh root@IP
+    ```
 
-```
-ssh root@IP
-```
+1. Get the install script, give it executable permissions, and run it:
 
-2. Get the install script, give it executable permissions, and run it:
+    ```
+    curl -o install.sh https://raw.githubusercontent.com/cerc-io/stack-orchestrator/main/scripts/quick-install-linux.sh
+    ```
+    ```
+    chmod +x install.sh
+    ```
+    ```
+    bash install.sh
+    ```
 
-```
-curl -o install.sh https://raw.githubusercontent.com/cerc-io/stack-orchestrator/main/scripts/quick-install-linux.sh
-```
-```
-chmod +x install.sh
-```
-```
-bash install.sh
-```
+1. Confirm docker was installed and activate the changes in `~/.profile`:
 
-3. Confirm docker was installed and activate the changes in `~/.profile`:
+    ```
+    docker run hello-world
+   ```
+   ```
+   source ~/.profile
+   ```
 
-```
-docker run hello-world
-```
-```
-source ~/.profile
-```
+1. Verify installation:
 
-4. Verify installation:
-
-```
-laconic-so version
-```
+    ```
+    laconic-so version
+    ```
 
 ## Setup the laconic fixturenet stack
 
 1. Get the repositories
 
-```
-laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
-```
+    ```
+    laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
+    ```
 
-2. Set this environment variable to the Laconic self-hosted Gitea instance:
+1. Build the containers:
 
-```
-export CERC_NPM_REGISTRY_URL=https://git.vdb.to/api/packages/cerc-io/npm/
-```
+    ```
+    laconic-so --stack fixturenet-laconic-loaded build-containers
+    ```
 
-3. Build the containers:
+    It's possible to run into an `ESOCKETTIMEDOUT` error, e.g., `error An unexpected error occurred: "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.11.3.tgz: ESOCKETTIMEDOUT"`. This may happen even if you have a great internet connection. In that case, re-run the `build-containers` command.
 
-```
-laconic-so --stack fixturenet-laconic-loaded build-containers
-```
 
-It's possible to run into an `ESOCKETTIMEDOUT` error, e.g., `error An unexpected error occurred: "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.11.3.tgz: ESOCKETTIMEDOUT"`. This may happen even if you have a great internet connection. In that case, re-run the `build-containers` command.
+1. Set this environment variable to your droplet's IP address or fully qualified DNS host name if it has one:
 
-4. Set this environment variable to your droplet's IP address:
+    ```
+    export BACKEND_ENDPOINT=http://<your-IP-or-hostname>:9473
+    ```
+    e.g.
+    ```
+    export BACKEND_ENDPOINT=http://my-test-server.example.com:9473
+    ```
 
-```
-export LACONIC_HOSTED_ENDPOINT=http://<your-IP>
-```
+1. Create a deployment directory for the stack:
+    ```
+    laconic-so --stack fixturenet-laconic-loaded deploy init --output laconic-loaded.spec --map-ports-to-host any-same --config LACONIC_HOSTED_ENDPOINT=$BACKEND_ENDPOINT
+    ```
+    ```
+    laconic-so --stack fixturenet-laconic-loaded deploy create --deployment-dir laconic-loaded-deployment --spec-file laconic-loaded.spec
+    ```
+2. Start the stack:
 
-5. Deploy the stack:
+    ```
+    laconic-so deployment --dir laconic-loaded-deployment start
+    ```
 
-```
-laconic-so --stack fixturenet-laconic-loaded deploy up
-```
+3. Check the logs:
 
-6. Check the logs:
+    ```
+    laconic-so deployment --dir laconic-loaded-deployment logs
+    ```
 
-```
-laconic-so --stack fixturenet-laconic-loaded deploy logs
-```
+    You'll see output from `laconicd` and the block height should be >1 to confirm it is running:
 
-You'll see output from `laconicd` and the block height should be >1 to confirm it is running:
+    ```
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:29PM INF indexed block exents height=12 module=txindex server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF Timed out dur=4976.960115 height=13 module=consensus round=0 server=node step=1
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4","parts":{"hash":"22411A20B7F14CDA33244420FBDDAF24450C0628C7A06034FF22DAC3699DDCC8","total":1}},"height":13,"pol_round":-1,"round":0,"signature":"DEuqnaQmvyYbUwckttJmgKdpRu6eVm9i+9rQ1pIrV2PidkMNdWRZBLdmNghkIrUzGbW8Xd7UVJxtLRmwRASgBg==","timestamp":"2023-04-18T21:30:01.49450663Z"} server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received complete proposal block hash=D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4 height=13 module=consensus server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF finalizing commit of block hash={} height=13 module=consensus num_txs=0 root=1A8CA1AF139CCC80EC007C6321D8A63A46A793386EE2EDF9A5CA0AB2C90728B7 server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF minted coins from module account amount=2059730459416582643aphoton from=mint module=x/bank
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF executed block height=13 module=state num_invalid_txs=0 num_valid_txs=0 server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF commit synced commit=436F6D6D697449447B5B363520313037203630203232372039352038352032303820313334203231392032303520313433203130372031343920313431203139203139322038362031323720362031383520323533203137362031333820313735203135392031383620323334203135382031323120313431203230342037335D3A447D
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF committed state app_hash=416B3CE35F55D086DBCD8F6B958D13C0567F06B9FDB08AAF9FBAEA9E798DCC49 height=13 module=state num_txs=0 server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF indexed block exents height=13 module=txindex server=node
+    ```
 
-```
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:29PM INF indexed block exents height=12 module=txindex server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF Timed out dur=4976.960115 height=13 module=consensus round=0 server=node step=1
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4","parts":{"hash":"22411A20B7F14CDA33244420FBDDAF24450C0628C7A06034FF22DAC3699DDCC8","total":1}},"height":13,"pol_round":-1,"round":0,"signature":"DEuqnaQmvyYbUwckttJmgKdpRu6eVm9i+9rQ1pIrV2PidkMNdWRZBLdmNghkIrUzGbW8Xd7UVJxtLRmwRASgBg==","timestamp":"2023-04-18T21:30:01.49450663Z"} server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received complete proposal block hash=D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4 height=13 module=consensus server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF finalizing commit of block hash={} height=13 module=consensus num_txs=0 root=1A8CA1AF139CCC80EC007C6321D8A63A46A793386EE2EDF9A5CA0AB2C90728B7 server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF minted coins from module account amount=2059730459416582643aphoton from=mint module=x/bank
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF executed block height=13 module=state num_invalid_txs=0 num_valid_txs=0 server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF commit synced commit=436F6D6D697449447B5B363520313037203630203232372039352038352032303820313334203231392032303520313433203130372031343920313431203139203139322038362031323720362031383520323533203137362031333820313735203135392031383620323334203135382031323120313431203230342037335D3A447D
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF committed state app_hash=416B3CE35F55D086DBCD8F6B958D13C0567F06B9FDB08AAF9FBAEA9E798DCC49 height=13 module=state num_txs=0 server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF indexed block exents height=13 module=txindex server=node
-```
+4. Confirm operation of the registry CLI:
 
-7. Confirm operation of the registry CLI:
+   ```
+   laconic-so deployment --dir laconic-loaded-deployment exec cli "laconic cns status"
+   ```
 
-```
-laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns status"
-```
-
-```
-{
-  "version": "0.3.0",
-  "node": {
-    "id": "4216af2ac9f68bda33a38803fc1b5c9559312c1d",
-    "network": "laconic_9000-1",
-    "moniker": "localtestnet"
-  },
-  "sync": {
-    "latest_block_hash": "1BDF4CB9AE2390DA65BCF997C83133C18014FCDDCAE03708488F0B56FCEEA429",
-    "latest_block_height": "5",
-    "latest_block_time": "2023-08-09 16:00:30.386903172 +0000 UTC",
-    "catching_up": false
-  },
-  "validator": {
-    "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
-    "voting_power": "1000000000000000"
-  },
-  "validators": [
-    {
-      "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
-      "voting_power": "1000000000000000",
-      "proposer_priority": "0"
-    }
-  ],
-  "num_peers": "0",
-  "peers": [],
-  "disk_usage": "292.0K"
-}
-```
+   ```
+   {
+     "version": "0.3.0",
+     "node": {
+       "id": "4216af2ac9f68bda33a38803fc1b5c9559312c1d",
+       "network": "laconic_9000-1",
+       "moniker": "localtestnet"
+     },
+     "sync": {
+       "latest_block_hash": "1BDF4CB9AE2390DA65BCF997C83133C18014FCDDCAE03708488F0B56FCEEA429",
+       "latest_block_height": "5",
+       "latest_block_time": "2023-08-09 16:00:30.386903172 +0000 UTC",
+       "catching_up": false
+     },
+     "validator": {
+       "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
+       "voting_power": "1000000000000000"
+     },
+     "validators": [
+       {
+         "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
+         "voting_power": "1000000000000000",
+         "proposer_priority": "0"
+       }
+     ],
+     "num_peers": "0",
+     "peers": [],
+     "disk_usage": "292.0K"
+   }
+   ```
 
 ## Configure Digital Ocean firewall
 
+(Note this step may not be necessary depending on the droplet image used)
+
 Let's open some ports.
 
 1. In the Digital Ocean web console, navigate to your droplet's main page. Select the "Networking" tab and scroll down to "Firewall".

docs/spec.md

@@ -1,6 +1,6 @@
 # Specification
 
-Note: this page is out of date (but still useful) - it will no longer be useful once stacks are [decoupled from the tool functionality](https://github.com/cerc-io/stack-orchestrator/issues/315).
+Note: this page is out of date (but still useful) - it will no longer be useful once stacks are [decoupled from the tool functionality](https://git.vdb.to/cerc-io/stack-orchestrator/issues/315).
 
 ## Implementation
 

requirements.txt

@@ -10,3 +10,4 @@ pydantic==1.10.9
 tomli==2.0.1
 validators==0.22.0
 kubernetes>=28.1.0
+humanfriendly>=10.0

scripts/cloud-init-dev-mode-install.yaml

@@ -41,4 +41,4 @@ runcmd:
   - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
   - systemctl enable docker
   - systemctl start docker
-  - git clone https://github.com/cerc-io/stack-orchestrator.git /home/ubuntu/stack-orchestrator
+  - git clone https://git.vdb.to/cerc-io/stack-orchestrator.git /home/ubuntu/stack-orchestrator

scripts/cloud-init-user-mode-install.yaml

@@ -31,5 +31,5 @@ runcmd:
   - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
   - systemctl enable docker
   - systemctl start docker
-  - curl -L -o /usr/local/bin/laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+  - curl -L -o /usr/local/bin/laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
   - chmod +x /usr/local/bin/laconic-so

scripts/quick-deploy-test.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Beginnings of a script to quickly spin up and test a deployment
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+if [[ -n "$1" ]]; then
+	stack_name=$1
+else
+	stack_name="test"
+fi
+spec_file_name="${stack_name}-spec.yml"
+deployment_dir_name="${stack_name}-deployment"
+rm -f ${spec_file_name}
+rm -rf ${deployment_dir_name}
+laconic-so --stack ${stack_name} deploy --deploy-to compose init --output ${spec_file_name}
+laconic-so --stack ${stack_name} deploy --deploy-to compose create --deployment-dir ${deployment_dir_name} --spec-file ${spec_file_name}
+#laconic-so deployment --dir ${deployment_dir_name} start
+#laconic-so deployment --dir ${deployment_dir_name} ps
+#laconic-so deployment --dir ${deployment_dir_name} stop

scripts/quick-install-linux.sh

@@ -137,7 +137,7 @@ fi
 echo "**************************************************************************************"
 echo "Installing laconic-so"
 # install latest `laconic-so`
-distribution_url=https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+distribution_url=https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
 install_filename=${install_dir}/laconic-so
 mkdir -p  ${install_dir}
 curl -L -o ${install_filename} ${distribution_url}

setup.py

@@ -13,7 +13,7 @@ setup(
     description='Orchestrates deployment of the Laconic stack',
     long_description=long_description,
     long_description_content_type="text/markdown",
-    url='https://github.com/cerc-io/stack-orchestrator',
+    url='https://git.vdb.to/cerc-io/stack-orchestrator',
     py_modules=['stack_orchestrator'],
     packages=find_packages(),
     install_requires=[requirements],

stack_orchestrator/build/build_containers.py

@@ -27,12 +27,13 @@ import subprocess
 import click
 import importlib.resources
 from pathlib import Path
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external
+from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, warn_exit
 from stack_orchestrator.base import get_npm_registry_url
 
 # TODO: find a place for this
 #    epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
 
+
 def make_container_build_env(dev_root_path: str,
                              container_build_dir: str,
                              debug: bool,
@@ -104,6 +105,9 @@ def process_container(stack: str,
         build_command = os.path.join(container_build_dir,
                                      "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
     if not dry_run:
+        # No PATH at all causes failures with podman.
+        if "PATH" not in container_build_env:
+            container_build_env["PATH"] = os.environ["PATH"]
         if verbose:
             print(f"Executing: {build_command} with environment: {container_build_env}")
         build_result = subprocess.run(build_command, shell=True, env=container_build_env)
@@ -119,6 +123,7 @@ def process_container(stack: str,
     else:
         print("Skipped")
 
+
 @click.command()
 @click.option('--include', help="only build these containers")
 @click.option('--exclude', help="don\'t build these containers")
@@ -159,6 +164,8 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
     containers_in_scope = []
     if stack:
         stack_config = get_parsed_stack_config(stack)
+        if "containers" not in stack_config or stack_config["containers"] is None:
+            warn_exit(f"stack {stack} does not define any containers")
         containers_in_scope = stack_config['containers']
     else:
         containers_in_scope = all_containers

stack_orchestrator/build/build_webapp.py

@@ -25,10 +25,11 @@ from decouple import config
 import click
 from pathlib import Path
 from stack_orchestrator.build import build_containers
+from stack_orchestrator.deploy.webapp.util import determine_base_container
 
 
 @click.command()
-@click.option('--base-container', default="cerc/nextjs-base")
+@click.option('--base-container')
 @click.option('--source-repo', help="directory containing the webapp to build", required=True)
 @click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
 @click.option("--extra-build-args", help="Supply extra arguments to build")
@@ -57,6 +58,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     if not quiet:
         print(f'Dev Root is: {dev_root_path}')
 
+    if not base_container:
+        base_container = determine_base_container(source_repo)
+
     # First build the base container.
     container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
                                                                     force_rebuild, extra_build_args)
@@ -64,13 +68,12 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
                                        verbose, dry_run, continue_on_error)
 
-
     # Now build the target webapp.  We use the same build script, but with a different Dockerfile and work dir.
     container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
     container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo)
     container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir,
-                                                                    base_container.replace("/", "-"),
-                                                                    "Dockerfile.webapp")
+                                                                          base_container.replace("/", "-"),
+                                                                          "Dockerfile.webapp")
     if not tag:
         webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
         container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"

stack_orchestrator/constants.py

@@ -27,6 +27,12 @@ kube_config_key = "kube-config"
 deploy_to_key = "deploy-to"
 network_key = "network"
 http_proxy_key = "http-proxy"
-image_resigtry_key = "image-registry"
+image_registry_key = "image-registry"
+configmaps_key = "configmaps"
+resources_key = "resources"
+volumes_key = "volumes"
+security_key = "security"
+annotations_key = "annotations"
+labels_key = "labels"
 kind_config_filename = "kind-config.yml"
 kube_config_filename = "kubeconfig.yml"

stack_orchestrator/data/compose/docker-compose-container-registry.yml

@@ -0,0 +1,13 @@
+services:
+  registry:
+    image: registry:2.8
+    restart: always
+    environment:
+      REGISTRY_LOG_LEVEL: ${REGISTRY_LOG_LEVEL}
+    volumes:
+      - registry-data:/var/lib/registry
+    ports:
+      - "5000"
+
+volumes:
+  registry-data:

stack_orchestrator/data/compose/docker-compose-grafana.yml

@@ -2,8 +2,10 @@ version: "3.7"
 
 services:
   grafana:
-    image: grafana/grafana
+    image: grafana/grafana:10.2.2
     restart: always
+    environment:
+      GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
     volumes:
       - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
       - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards

stack_orchestrator/data/compose/docker-compose-mars-v2.yml

@@ -0,0 +1,12 @@
+version: "3.2"
+
+services:
+  mars:
+    image: cerc/mars-v2:local
+    restart: always
+    ports:
+      - "3000:3000"
+    environment:
+      - URL_OSMOSIS_REST=https://lcd-osmosis.blockapsis.com
+      - URL_OSMOSIS_RPC=https://rpc-osmosis.blockapsis.com
+      - WALLET_CONNECT_ID=0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x

stack_orchestrator/data/compose/docker-compose-mars.yml

@@ -0,0 +1,20 @@
+version: "3.2"
+
+services:
+  mars:
+    image: cerc/mars:local
+    restart: always
+    ports:
+      - "3000:3000"
+    environment:
+      - URL_OSMOSIS_GQL=https://osmosis-node.marsprotocol.io/GGSFGSFGFG34/osmosis-hive-front/graphql
+      - URL_OSMOSIS_REST=https://lcd-osmosis.blockapsis.com
+      - URL_OSMOSIS_RPC=https://rpc-osmosis.blockapsis.com
+      - URL_NEUTRON_GQL=https://neutron.rpc.p2p.world/qgrnU6PsQZA8F9S5Fb8Fn3tV3kXmMBl2M9bcc9jWLjQy8p/hive/graphql
+      - URL_NEUTRON_REST=https://rest-kralum.neutron-1.neutron.org
+      - URL_NEUTRON_RPC=https://rpc-kralum.neutron-1.neutron.org
+      - URL_NEUTRON_TEST_GQL=https://testnet-neutron-gql.marsprotocol.io/graphql
+      - URL_NEUTRON_TEST_REST=https://rest-palvus.pion-1.ntrn.tech
+      - URL_NEUTRON_TEST_RPC=https://rpc-palvus.pion-1.ntrn.tech
+      - WALLET_CONNECT_ID=0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x
+

stack_orchestrator/data/compose/docker-compose-node-exporter.yml

@@ -5,12 +5,11 @@ services:
     image: prom/node-exporter:latest
     restart: unless-stopped
     command:
+      - '--web.listen-address=:9100'
       - '--path.rootfs=/host'
       - '--collector.systemd'
       - '--collector.processes'
     network_mode: host
     pid: host
-    ports:
-      - 9100
     volumes:
       - '/:/host:ro,rslave'

stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml

@@ -6,6 +6,7 @@ services:
     restart: on-failure
     environment:
       - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL}
+      - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438
       - NEXT_PUBLIC_BASEPATH=/apps/osmosis
       - NEXT_PUBLIC_URBIT_DEPLOYMENT=true
     working_dir: /app/packages/web

stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml

@@ -6,6 +6,7 @@ services:
     restart: on-failure
     environment:
       - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL}
+      - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438
     working_dir: /app/packages/web
     command: ["./build-app.sh"]
     volumes:

stack_orchestrator/data/compose/docker-compose-ping-pub.yml

@@ -0,0 +1,8 @@
+version: "3.2"
+
+services:
+  ping-pub:
+    image: cerc/ping-pub:local
+    restart: always
+    ports:
+      - "5173:5173"

stack_orchestrator/data/compose/docker-compose-prom-server.yml

@@ -2,7 +2,7 @@ version: "3.7"
 
 services:
   prometheus:
-    image: prom/prometheus
+    image: prom/prometheus:v2.49.1
     restart: always
     volumes:
       - ../config/monitoring/prometheus:/etc/prometheus

stack_orchestrator/data/compose/docker-compose-snowballtools-base-backend.yml

@@ -0,0 +1,13 @@
+services:
+  snowballtools-base-backend:
+    image: cerc/snowballtools-base-backend:local
+    restart: always
+    volumes:
+      - data:/data
+      - config:/config:ro
+    ports:
+      - 8000
+
+volumes:
+  data:
+  config:

stack_orchestrator/data/compose/docker-compose-test-database.yml

@@ -0,0 +1,20 @@
+services:
+
+  database:
+    image: cerc/test-database-container:local
+    restart: always
+    volumes:
+      - db-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: "test-user"
+      POSTGRES_DB: "test-db"
+      POSTGRES_PASSWORD: "password"
+      POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C"
+    ports:
+      - "5432"
+    
+  test-client:
+    image: cerc/test-database-client:local
+
+volumes:
+  db-data:

stack_orchestrator/data/compose/docker-compose-test.yml

@@ -5,10 +5,16 @@ services:
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_TEST_PARAM_1: ${CERC_TEST_PARAM_1:-FAILED}
+      CERC_TEST_PARAM_2: "CERC_TEST_PARAM_2_VALUE"
+      CERC_TEST_PARAM_3: ${CERC_TEST_PARAM_3:-FAILED}
     volumes:
-      - test-data:/data
+      - test-data-bind:/data
+      - test-data-auto:/data2
+      - test-config:/config:ro
     ports:
       - "80"
 
 volumes:
-  test-data:
+  test-data-bind:
+  test-data-auto:
+  test-config:

stack_orchestrator/data/compose/docker-compose-webapp-template.yml

@@ -5,4 +5,4 @@ services:
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
     ports:
-      - "3000"
+      - "80"

stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/contactpoints.yml

@@ -0,0 +1,14 @@
+# https://www.clever-cloud.com/blog/features/2021/12/03/slack-alerts-for-grafana/
+
+apiVersion: 1
+
+contactPoints:
+  - orgId: 1
+    name: SlackNotifier
+    receivers:
+      - uid: a71b06e3-58b6-41fe-af65-fbbb29653951
+        type: slack
+        settings:
+          # Slack hook URL (see https://api.slack.com/messaging/webhooks)
+          url: <YOUR_SLACK_HOOK_URL>
+        disableResolveMessage: false

stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/policies.yml

@@ -0,0 +1,15 @@
+# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-notification-policy/
+
+apiVersion: 1
+
+policies:
+  - orgId: 1
+    receiver: grafana-default-email
+    group_by:
+      - grafana_folder
+      - alertname
+    routes:
+      - receiver: SlackNotifier
+        object_matchers:
+          # Add matchers below
+          # - ['grafana_folder', '=', 'MyAlerts']

stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml

@@ -0,0 +1,933 @@
+# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-grafana-managed-rule/
+
+apiVersion: 1
+
+groups:
+  - orgId: 1
+    name: watcher
+    folder: WatcherAlerts
+    interval: 30s
+    rules:
+      # Azimuth
+      - uid: azimuth_diff_external
+        title: azimuth_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: censures_diff_external
+        title: censures_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: claims_diff_external
+        title: claims_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: conditional_star_release_diff_external
+        title: conditional_star_release_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: delegated_sending_diff_external
+        title: delegated_sending_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: ecliptic_diff_external
+        title: ecliptic_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: linear_star_release_diff_external
+        title: linear_star_release_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: polls_diff_external
+        title: polls_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+
+      # Sushi
+      - uid: sushiswap_diff_external
+        title: sushiswap_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: merkl_sushiswap_diff_external
+        title: merkl_sushiswap_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false

stack_orchestrator/data/container-build/cerc-act-runner-task-executor/build.sh

@@ -2,4 +2,6 @@
 # Build a local version of the task executor for act-runner
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/act-runner/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR}
+
+cd ${CERC_REPO_BASE_DIR}/hosting/act-runner
+docker build -t cerc/act-runner-task-executor:local -f Dockerfile.task-executor ${build_command_args} .

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile

@@ -10,9 +10,10 @@ COPY genesis /opt/genesis
 COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
 COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
 COPY --from=ethgen /apps /apps
-RUN cd /apps/el-gen && pip3 install -r requirements.txt
+RUN cd /apps/el-gen && pip3 install --break-system-packages -r requirements.txt
 # web3==5.24.0 used by el-gen is broken on python 3.11
-RUN pip3 install --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "typing-extensions"
 
 # Build genesis config
 RUN apk add --no-cache make bash envsubst jq

stack_orchestrator/data/container-build/cerc-mars-v2/build.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build the mars-v2 image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+docker build -t cerc/mars-v2:local -f ${CERC_REPO_BASE_DIR}/mars-v2-frontend/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/mars-v2-frontend

stack_orchestrator/data/container-build/cerc-mars/build.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build the mars image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+docker build -t cerc/mars:local -f ${CERC_REPO_BASE_DIR}/mars-interface/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/mars-interface

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -30,13 +30,13 @@ RUN \
 
 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base
+    && apt-get -y install --no-install-recommends jq gettext-base procps
 
 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"
 
 # Expose port for http
-EXPOSE 3000
+EXPOSE 80
 
 COPY /scripts /scripts
 

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh

@@ -33,8 +33,8 @@ if [ -f ".env" ]; then
   rm -f $TMP_ENV
 fi
 
-for f in $(find "$TRG_DIR" -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do
-  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},();]' '\n' | egrep -o '^"CERC_RUNTIME_ENV_[^\"]+"'); do
+for f in $(find . -type f \( -regex '.*.html?' -or -regex ".*.[tj]s\(x\|on\)?$" \) | grep -v 'node_modules' | grep -v '.git'); do
+  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '["/\\{},();]' '\n' | tr -s "[']" '\n' | egrep -o -e '^CERC_RUNTIME_ENV_.+$' -e '^LACONIC_HOSTED_CONFIG_.+$'); do
     orig_name=$(echo -n "${e}" | sed 's/"//g')
     cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
     cur_val=$(echo -n "\$${cur_name}" | envsubst)

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -21,6 +21,11 @@ WORK_DIR="${1:-/app}"
 
 cd "${WORK_DIR}" || exit 1
 
+# If this file doesn't exist at all, we'll get errors below.
+if [ ! -f "next.config.js" ]; then
+  touch next.config.js
+fi
+
 if [ ! -f "next.config.dist" ]; then
   cp next.config.js next.config.dist
 fi

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -58,4 +58,4 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
   fi
 fi
 
-$CERC_BUILD_TOOL start . -p ${CERC_LISTEN_PORT:-3000}
+$CERC_BUILD_TOOL start . -- -p ${CERC_LISTEN_PORT:-80}

stack_orchestrator/data/container-build/cerc-ping-pub/build.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Build the ping pub image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+docker build -t cerc/ping-pub:local ${build_command_args} -f $CERC_REPO_BASE_DIR/explorer/Dockerfile $CERC_REPO_BASE_DIR/explorer

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/Dockerfile

@@ -0,0 +1,6 @@
+FROM cerc/snowballtools-base-backend-base:local
+
+WORKDIR /app/packages/backend
+COPY run.sh .
+
+ENTRYPOINT ["./run.sh"]

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/Dockerfile-base

@@ -0,0 +1,26 @@
+FROM ubuntu:22.04 as builder
+
+RUN apt update && \
+      apt install -y --no-install-recommends --no-install-suggests \
+      ca-certificates curl gnupg
+
+# Node
+ARG NODE_MAJOR=20
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
+      echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
+      apt update && apt install -y nodejs
+
+# npm setup
+RUN npm config set @cerc-io:registry https://git.vdb.to/api/packages/cerc-io/npm/ && npm install -g yarn
+
+COPY . /app/
+WORKDIR /app/
+
+RUN find . -name 'node_modules' | xargs -n1 rm -rf
+RUN yarn && yarn build --ignore frontend
+
+FROM cerc/webapp-base:local
+
+COPY --from=builder /app /app
+
+WORKDIR /app/packages/backend

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/build.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# Build cerc/webapp-deployer-backend
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/snowballtools-base-backend-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile-base ${CERC_REPO_BASE_DIR}/snowballtools-base
+docker build -t cerc/snowballtools-base-backend:local ${build_command_args} ${SCRIPT_DIR} 

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/run.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+
+LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
+if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
+  if [ -f "/config/laconic-hosted-config.yml" ]; then
+    LACONIC_HOSTED_CONFIG_FILE="/config/laconic-hosted-config.yml"
+  elif [ -f "/config/config.yml" ]; then
+    LACONIC_HOSTED_CONFIG_FILE="/config/config.yml"
+  fi
+fi
+
+if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
+  /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "`pwd`/dist"
+fi
+
+/scripts/apply-runtime-env.sh "`pwd`/dist"
+
+yarn start

stack_orchestrator/data/container-build/cerc-test-container/run.sh

@@ -1,21 +1,67 @@
 #!/usr/bin/env bash
 set -e
+
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
-# Test if the container's filesystem is old (run previously) or new
-EXISTSFILENAME=/data/exists
+
 echo "Test container starting"
-if [[ -f "$EXISTSFILENAME" ]];
-then
-    TIMESTAMP=`cat $EXISTSFILENAME`
-    echo "Filesystem is old, created: $TIMESTAMP" 
+
+DATA_DEVICE=$(df | grep "/data$" | awk '{ print $1 }')
+if [[ -n "$DATA_DEVICE" ]]; then
+  echo "/data: MOUNTED dev=${DATA_DEVICE}"
 else
-    echo "Filesystem is fresh"
-    echo `date` > $EXISTSFILENAME
+  echo "/data: not mounted"
 fi
+
+DATA2_DEVICE=$(df | grep "/data2$" | awk '{ print $1 }')
+if [[ -n "$DATA_DEVICE" ]]; then
+  echo "/data2: MOUNTED dev=${DATA2_DEVICE}"
+else
+  echo "/data2: not mounted"
+fi
+
+# Test if the container's filesystem is old (run previously) or new
+for d in /data /data2; do
+  if [[ -f "$d/exists" ]];
+  then
+      TIMESTAMP=`cat $d/exists`
+      echo "$d filesystem is old, created: $TIMESTAMP"
+  else
+      echo "$d filesystem is fresh"
+      echo `date` > $d/exists
+  fi
+done
+
 if [ -n "$CERC_TEST_PARAM_1" ]; then
   echo "Test-param-1: ${CERC_TEST_PARAM_1}"
 fi
+if [ -n "$CERC_TEST_PARAM_2" ]; then
+  echo "Test-param-2: ${CERC_TEST_PARAM_2}"
+fi
+if [ -n "$CERC_TEST_PARAM_3" ]; then
+  echo "Test-param-3: ${CERC_TEST_PARAM_3}"
+fi
+if [ -n "$CERC_TEST_PARAM_4" ]; then
+  echo "Test-param-4: ${CERC_TEST_PARAM_4}"
+fi
+if [ -n "$CERC_TEST_PARAM_5" ]; then
+  echo "Test-param-5: ${CERC_TEST_PARAM_5}"
+fi
+
+if [ -d "/config" ]; then
+  echo "/config: EXISTS"
+  for f in /config/*; do
+    if [[ -f "$f" ]] || [[ -L "$f" ]]; then
+      echo "$f:"
+      cat "$f"
+      echo ""
+      echo ""
+    fi
+  done
+else
+  echo "/config: does NOT EXIST"
+fi
+
 # Run nginx which will block here forever
 /usr/sbin/nginx -g "daemon off;"

stack_orchestrator/data/container-build/cerc-test-database-client/Dockerfile

@@ -0,0 +1,12 @@
+FROM ubuntu:latest
+
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive && export DEBCONF_NOWARNINGS="yes" && \
+    apt-get install -y software-properties-common && \
+    apt-get install -y postgresql-client && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+EXPOSE 80
+
+COPY run.sh /app/run.sh
+
+ENTRYPOINT ["/app/run.sh"]

stack_orchestrator/data/container-build/cerc-test-database-client/build.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Build cerc/test-container
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+docker build -t cerc/test-database-client:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR

stack_orchestrator/data/container-build/cerc-test-database-client/run.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# TODO derive this from config
+database_url="postgresql://test-user:password@localhost:5432/test-db"
+psql_command="psql ${database_url}"
+program_name="Database test client:"
+
+wait_for_database_up () {
+    for i in {1..50}
+    do
+        ${psql_command} -c "select 1;"
+        psql_succeeded=$?
+        if [[ ${psql_succeeded} == 0 ]]; then
+            # if ready, return
+            echo "${program_name} database up"
+            return
+        else
+            # if not ready, wait
+            echo "${program_name} waiting for database: ${i}"
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "${program_name} waiting for database: FAILED"
+    exit 1
+}
+
+# Used to synchronize with the test runner
+notify_test_complete () {
+    echo "${program_name} test complete"
+}
+
+does_test_data_exist () {
+    query_result=$(${psql_command} -t -c "select count(*) from test_table_1 where key_column = 'test_key_1';" | head -1 | tr -d ' ')
+    if [[ "${query_result}" == "1" ]]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+create_test_data () {
+    ${psql_command} -c "create table test_table_1 (key_column text, value_column text, primary key(key_column));"
+    ${psql_command} -c "insert into test_table_1 values ('test_key_1', 'test_value_1');"
+}
+
+wait_forever() {
+    # Loop to keep docker/k8s happy since this is the container entrypoint
+    while :; do sleep 600; done
+}
+
+wait_for_database_up
+
+# Check if the test database content exists already
+if does_test_data_exist; then
+    # If so, log saying so. Test harness will look for this log output
+    echo "${program_name} test data already exists"
+else
+    # Otherwise log saying the content was not present
+    echo "${program_name} test data does not exist"
+    echo "${program_name} creating test data"
+    # then create it
+    create_test_data
+fi
+
+notify_test_complete
+wait_forever

stack_orchestrator/data/container-build/cerc-test-database-container/Dockerfile

@@ -0,0 +1,3 @@
+FROM postgres:16-bullseye
+
+EXPOSE 5432

stack_orchestrator/data/container-build/cerc-test-database-container/build.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Build cerc/test-container
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+docker build -t cerc/test-database-container:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR

stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile

@@ -1,6 +1,6 @@
 # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
 # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT=18-bullseye
+ARG VARIANT=20-bullseye
 FROM node:${VARIANT}
 
 ARG USERNAME=node
@@ -28,7 +28,7 @@ RUN \
 
 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq
+    && apt-get -y install --no-install-recommends jq gettext-base
 
 # [Optional] Uncomment if you want to install an additional version of node using nvm
 # ARG EXTRA_NODE_VERSION=10
@@ -37,15 +37,12 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 # We do this to get a yq binary from the published container, for the correct architecture we're building here
 COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
 
-RUN mkdir -p /scripts
-COPY ./apply-webapp-config.sh /scripts
-COPY ./start-serving-app.sh /scripts
+COPY scripts /scripts
 
 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"
 
 RUN mkdir -p /config
-COPY ./config.yml /config
 
 # Install simple web server for now (use nginx perhaps later)
 RUN yarn global add http-server

stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp

@@ -0,0 +1,11 @@
+FROM cerc/webapp-base:local as builder
+
+ARG CERC_BUILD_TOOL
+
+WORKDIR /app
+COPY . .
+RUN rm -rf node_modules build .next*
+RUN /scripts/build-app.sh /app build /data
+
+FROM cerc/webapp-base:local
+COPY --from=builder /data /data

stack_orchestrator/data/container-build/cerc-webapp-base/build.sh

@@ -1,9 +1,29 @@
 #!/usr/bin/env bash
-# Build cerc/laconic-registry-cli
+# Build cerc/webapp-base
 
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 
 # See: https://stackoverflow.com/a/246128/1701505
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-docker build -t cerc/webapp-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile ${SCRIPT_DIR}
+CERC_CONTAINER_BUILD_WORK_DIR=${CERC_CONTAINER_BUILD_WORK_DIR:-$SCRIPT_DIR}
+CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/Dockerfile}
+CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/webapp-base:local}
+
+docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
+
+if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/webapp-base:local" ]; then
+  cat <<EOF
+
+#################################################################
+
+Built host container for $CERC_CONTAINER_BUILD_WORK_DIR with tag:
+
+    $CERC_CONTAINER_BUILD_TAG
+
+To test locally run:
+
+    laconic-so run-webapp --image $CERC_CONTAINER_BUILD_TAG --env-file /path/to/environment.env
+
+EOF
+fi

stack_orchestrator/data/container-build/cerc-webapp-base/config.yml

@@ -1 +0,0 @@
-# Put config here.

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/apply-runtime-env.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+
+WORK_DIR="${1:-./}"
+
+cd "${WORK_DIR}" || exit 1
+
+if [ -f ".env" ]; then
+  TMP_ENV=`mktemp`
+  declare -px > $TMP_ENV
+  set -a
+  source .env
+  source $TMP_ENV
+  set +a
+  rm -f $TMP_ENV
+fi
+
+for f in $(find . -type f \( -regex '.*.html?' -or -regex ".*.[tj]s\(x\|on\)?$" \) | grep -v 'node_modules' | grep -v '.git'); do
+  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '["/\\{},();]' '\n' | tr -s "[']" '\n' | egrep -o -e '^CERC_RUNTIME_ENV_.+$' -e '^LACONIC_HOSTED_CONFIG_.+$'); do
+    orig_name=$(echo -n "${e}" | sed 's/"//g')
+    cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
+    cur_val=$(echo -n "\$${cur_name}" | envsubst)
+    if [ "$CERC_RETAIN_ENV_QUOTES" != "true" ]; then
+      cur_val=$(sed "s/^[\"']//" <<< "$cur_val" | sed "s/[\"']//")
+    fi
+    esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val")
+    echo "$f: $cur_name=$cur_val"
+    sed -i "s/$orig_name/$esc_val/g" $f
+  done
+done

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+
+CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+WORK_DIR="${1:-/app}"
+OUTPUT_DIR="${2:-build}"
+DEST_DIR="${3:-/data}"
+
+if [ -f "${WORK_DIR}/build-webapp.sh" ]; then
+  echo "Building webapp with ${WORK_DIR}/build-webapp.sh ..."
+  cd "${WORK_DIR}" || exit 1
+
+  rm -rf "${DEST_DIR}"
+  ./build-webapp.sh "${DEST_DIR}" || exit 1
+elif [ -f "${WORK_DIR}/package.json" ]; then
+  echo "Building node-based webapp ..."
+  cd "${WORK_DIR}" || exit 1
+
+  if [ -z "$CERC_BUILD_TOOL" ]; then
+    if [ -f "yarn.lock" ]; then
+      CERC_BUILD_TOOL=yarn
+    else
+      CERC_BUILD_TOOL=npm
+    fi
+  fi
+
+  $CERC_BUILD_TOOL install || exit 1
+  $CERC_BUILD_TOOL build || exit 1
+
+  rm -rf "${DEST_DIR}"
+  mv "${WORK_DIR}/${OUTPUT_DIR}" "${DEST_DIR}"
+else
+  echo "Copying static app ..."
+  mv "${WORK_DIR}" "${DEST_DIR}"
+fi
+
+# One special fix ...
+cd "${DEST_DIR}"
+for f in $(find . -type f -name '*.htm*'); do
+  sed -i -e 's#/LACONIC_HOSTED_CONFIG_homepage/#LACONIC_HOSTED_CONFIG_homepage/#g' "$f"
+done
+
+exit 0

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+
+CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}"
+CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}"
+
+if [ "true" == "$CERC_ENABLE_CORS" ]; then
+  CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --cors"
+fi
+
+LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
+if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
+  if [ -f "/config/laconic-hosted-config.yml" ]; then
+    LACONIC_HOSTED_CONFIG_FILE="/config/laconic-hosted-config.yml"
+  elif [ -f "/config/config.yml" ]; then
+    LACONIC_HOSTED_CONFIG_FILE="/config/config.yml"
+  fi
+fi
+
+if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
+  /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE ${CERC_WEBAPP_FILES_DIR}
+fi
+
+/scripts/apply-runtime-env.sh ${CERC_WEBAPP_FILES_DIR}
+http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT:-80} ${CERC_WEBAPP_FILES_DIR}

stack_orchestrator/data/container-build/cerc-webapp-base/start-serving-app.sh

@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-    set -x
-fi
-
-CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}"
-
-/scripts/apply-webapp-config.sh /config/config.yml ${CERC_WEBAPP_FILES_DIR}
-http-server -p 80 ${CERC_WEBAPP_FILES_DIR}

stack_orchestrator/data/container-build/cerc-webapp-deployer-backend/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Build cerc/webapp-deployer-backend
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/webapp-deployer-backend:local ${build_command_args} ${CERC_REPO_BASE_DIR}/webapp-deployment-status-api

stack_orchestrator/data/k8s/components/ingress/ingress-nginx-kind-deploy.yaml

@@ -0,0 +1,673 @@
+# from: https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
+# via: https://kind.sigs.k8s.io/docs/user/ingress/#ingress-nginx
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+  name: ingress-nginx
+---
+apiVersion: v1
+automountServiceAccountToken: true
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx
+  namespace: ingress-nginx
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx
+  namespace: ingress-nginx
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - pods
+  - secrets
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses/status
+  verbs:
+  - update
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingressclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - coordination.k8s.io
+  resourceNames:
+  - ingress-nginx-leader
+  resources:
+  - leases
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - list
+  - watch
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - endpoints
+  - nodes
+  - pods
+  - secrets
+  - namespaces
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses/status
+  verbs:
+  - update
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingressclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - list
+  - watch
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+rules:
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - validatingwebhookconfigurations
+  verbs:
+  - get
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx
+  namespace: ingress-nginx
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ingress-nginx
+subjects:
+- kind: ServiceAccount
+  name: ingress-nginx
+  namespace: ingress-nginx
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ingress-nginx-admission
+subjects:
+- kind: ServiceAccount
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ingress-nginx
+subjects:
+- kind: ServiceAccount
+  name: ingress-nginx
+  namespace: ingress-nginx
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ingress-nginx-admission
+subjects:
+- kind: ServiceAccount
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+---
+apiVersion: v1
+data:
+  allow-snippet-annotations: "false"
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+spec:
+  ipFamilies:
+  - IPv4
+  ipFamilyPolicy: SingleStack
+  ports:
+  - appProtocol: http
+    name: http
+    port: 80
+    protocol: TCP
+    targetPort: http
+  - appProtocol: https
+    name: https
+    port: 443
+    protocol: TCP
+    targetPort: https
+  selector:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+  type: NodePort
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-controller-admission
+  namespace: ingress-nginx
+spec:
+  ports:
+  - appProtocol: https
+    name: https-webhook
+    port: 443
+    targetPort: webhook
+  selector:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+spec:
+  minReadySeconds: 0
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: controller
+      app.kubernetes.io/instance: ingress-nginx
+      app.kubernetes.io/name: ingress-nginx
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: controller
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/part-of: ingress-nginx
+        app.kubernetes.io/version: 1.9.6
+    spec:
+      containers:
+      - args:
+        - /nginx-ingress-controller
+        - --election-id=ingress-nginx-leader
+        - --controller-class=k8s.io/ingress-nginx
+        - --ingress-class=nginx
+        - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
+        - --validating-webhook=:8443
+        - --validating-webhook-certificate=/usr/local/certificates/cert
+        - --validating-webhook-key=/usr/local/certificates/key
+        - --watch-ingress-without-class=true
+        - --publish-status-address=localhost
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: LD_PRELOAD
+          value: /usr/local/lib/libmimalloc.so
+        image: registry.k8s.io/ingress-nginx/controller:v1.9.6@sha256:1405cc613bd95b2c6edd8b2a152510ae91c7e62aea4698500d23b2145960ab9c
+        imagePullPolicy: IfNotPresent
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - /wait-shutdown
+        livenessProbe:
+          failureThreshold: 5
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+        name: controller
+        ports:
+        - containerPort: 80
+          hostPort: 80
+          name: http
+          protocol: TCP
+        - containerPort: 443
+          hostPort: 443
+          name: https
+          protocol: TCP
+        - containerPort: 8443
+          name: webhook
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+        resources:
+          requests:
+            cpu: 100m
+            memory: 90Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - ALL
+          readOnlyRootFilesystem: false
+          runAsNonRoot: true
+          runAsUser: 101
+          seccompProfile:
+            type: RuntimeDefault
+        volumeMounts:
+        - mountPath: /usr/local/certificates/
+          name: webhook-cert
+          readOnly: true
+      dnsPolicy: ClusterFirst
+      nodeSelector:
+        ingress-ready: "true"
+        kubernetes.io/os: linux
+      serviceAccountName: ingress-nginx
+      terminationGracePeriodSeconds: 0
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+        operator: Equal
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+        operator: Equal
+      volumes:
+      - name: webhook-cert
+        secret:
+          secretName: ingress-nginx-admission
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission-create
+  namespace: ingress-nginx
+spec:
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: admission-webhook
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/part-of: ingress-nginx
+        app.kubernetes.io/version: 1.9.6
+      name: ingress-nginx-admission-create
+    spec:
+      containers:
+      - args:
+        - create
+        - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc
+        - --namespace=$(POD_NAMESPACE)
+        - --secret-name=ingress-nginx-admission
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20231226-1a7112e06@sha256:25d6a5f11211cc5c3f9f2bf552b585374af287b4debf693cacbe2da47daa5084
+        imagePullPolicy: IfNotPresent
+        name: create
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 65532
+          seccompProfile:
+            type: RuntimeDefault
+      nodeSelector:
+        kubernetes.io/os: linux
+      restartPolicy: OnFailure
+      serviceAccountName: ingress-nginx-admission
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission-patch
+  namespace: ingress-nginx
+spec:
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: admission-webhook
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/part-of: ingress-nginx
+        app.kubernetes.io/version: 1.9.6
+      name: ingress-nginx-admission-patch
+    spec:
+      containers:
+      - args:
+        - patch
+        - --webhook-name=ingress-nginx-admission
+        - --namespace=$(POD_NAMESPACE)
+        - --patch-mutating=false
+        - --secret-name=ingress-nginx-admission
+        - --patch-failure-policy=Fail
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20231226-1a7112e06@sha256:25d6a5f11211cc5c3f9f2bf552b585374af287b4debf693cacbe2da47daa5084
+        imagePullPolicy: IfNotPresent
+        name: patch
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 65532
+          seccompProfile:
+            type: RuntimeDefault
+      nodeSelector:
+        kubernetes.io/os: linux
+      restartPolicy: OnFailure
+      serviceAccountName: ingress-nginx-admission
+---
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: nginx
+spec:
+  controller: k8s.io/ingress-nginx
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-webhook
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+    app.kubernetes.io/version: 1.9.6
+  name: ingress-nginx-admission
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: ingress-nginx-controller-admission
+      namespace: ingress-nginx
+      path: /networking/v1/ingresses
+  failurePolicy: Fail
+  matchPolicy: Equivalent
+  name: validate.nginx.ingress.kubernetes.io
+  rules:
+  - apiGroups:
+    - networking.k8s.io
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - ingresses
+  sideEffects: None

stack_orchestrator/data/stacks/act-runner/stack.yml

@@ -3,7 +3,7 @@ name: act-runner
 description: "Local act-runner"
 repos:
   - git.vdb.to/cerc-io/hosting
-  - gitea.com/gitea/act_runner
+  - gitea.com/telackey/act_runner@telackey/entrypoint
 containers:
   - cerc/act-runner
   - cerc/act-runner-task-executor

stack_orchestrator/data/stacks/container-registry/README.md

@@ -0,0 +1,3 @@
+# Container Registry Stack
+
+Host a container image registry

stack_orchestrator/data/stacks/container-registry/stack.yml

@@ -0,0 +1,5 @@
+version: "1.0"
+name: container-registry
+description: "Container registry stack"
+pods:
+  - container-registry

stack_orchestrator/data/stacks/fixturenet-eth/README.md

@@ -1,6 +1,6 @@
 # fixturenet-eth
 
-Instructions for deploying a local a geth + lighthouse blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator)):
+Instructions for deploying a local a geth + lighthouse blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator (the installation of which is covered [here](https://git.vdb.to/cerc-io/stack-orchestrator)):
 
 ## Clone required repositories
 

stack_orchestrator/data/stacks/fixturenet-laconic-loaded/README.md

@@ -7,11 +7,11 @@ Instructions for deploying a local Laconic blockchain "fixturenet" for developme
 **Note:** For building some NPMs, access to the @lirewine repositories is required. If you don't have access, see [this tutorial](/docs/laconicd-fixturenet.md) to run this stack
 
 ## 1. Install Laconic Stack Orchestrator
-Installation is covered in detail [here](https://github.com/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as:
+Installation is covered in detail [here](https://git.vdb.to/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as:
 ```
 $ mkdir my-working-dir
 $ cd my-working-dir
-$ curl -L -o ./laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+$ curl -L -o ./laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
 $ chmod +x ./laconic-so
 $ export PATH=$PATH:$(pwd)  # Or move laconic-so to ~/bin or your favorite on-path directory
 ```

stack_orchestrator/data/stacks/fixturenet-laconicd/README.md

@@ -3,11 +3,11 @@
 Instructions for deploying a local Laconic blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator.
 
 ## 1. Install Laconic Stack Orchestrator
-Installation is covered in detail [here](https://github.com/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as:
+Installation is covered in detail [here](https://git.vdb.to/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as:
 ```
 $ mkdir my-working-dir
 $ cd my-working-dir
-$ curl -L -o ./laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+$ curl -L -o ./laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
 $ chmod +x ./laconic-so
 $ export PATH=$PATH:$(pwd)  # Or move laconic-so to ~/bin or your favorite on-path directory
 ```

stack_orchestrator/data/stacks/mainnet-eth/README.md

@@ -18,7 +18,7 @@ $ laconic-so --stack mainnet-eth build-containers
 
 ```
 $ laconic-so --stack mainnet-eth deploy init --map-ports-to-host any-same --output mainnet-eth-spec.yml
-$ laconic-so deploy create --spec-file mainnet-eth-spec.yml --deployment-dir mainnet-eth-deployment
+$ laconic-so deploy --stack mainnet-eth create --spec-file mainnet-eth-spec.yml --deployment-dir mainnet-eth-deployment
 ```
 ## Start the stack
 ```

stack_orchestrator/data/stacks/mars-v2/README.md

@@ -0,0 +1,16 @@
+# mars
+
+On a fresh Digital Ocean droplet with Ubuntu:
+
+```
+git clone https://github.com/cerc-io/stack-orchestrator
+cd stack-orchestrator
+./scripts/quick-install-linux.sh
+```
+Read and follow the instructions output from the above output to complete installation, then:
+
+```
+laconic-so --stack mars-v2 setup-repositories
+laconic-so --stack mars-v2 build-containers
+laconic-so --stack mars-v2 deploy up
+```

stack_orchestrator/data/stacks/mars-v2/stack.yml

@@ -0,0 +1,8 @@
+version: "0.1"
+name: mars-v2
+repos:
+  - github.com/mars-protocol/mars-v2-frontend
+containers:
+  - cerc/mars-v2
+pods:
+  - mars-v2

stack_orchestrator/data/stacks/mars/README.md

@@ -0,0 +1,16 @@
+# mars
+
+On a fresh Digital Ocean droplet with Ubuntu:
+
+```
+git clone https://github.com/cerc-io/stack-orchestrator
+cd stack-orchestrator
+./scripts/quick-install-linux.sh
+```
+Read and follow the instructions output from the above output to complete installation, then:
+
+```
+laconic-so --stack mars setup-repositories
+laconic-so --stack mars build-containers
+laconic-so --stack mars deploy up
+```

stack_orchestrator/data/stacks/mars/stack.yml

@@ -0,0 +1,8 @@
+version: "0.1"
+name: mars
+repos:
+  - github.com/cerc-io/mars-interface
+containers:
+  - cerc/mars
+pods:
+  - mars

stack_orchestrator/data/stacks/mobymask/README.md

@@ -4,7 +4,7 @@ The MobyMask watcher is a Laconic Network component that provides efficient acce
 
 ## Deploy the MobyMask Watcher
 
-The instructions below show how to deploy a MobyMask watcher using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator#install)).
+The instructions below show how to deploy a MobyMask watcher using laconic-stack-orchestrator (the installation of which is covered [here](https://git.vdb.to/cerc-io/stack-orchestrator#install)).
 
 This deployment expects that ipld-eth-server's endpoints are available on the local machine at http://ipld-eth-server.example.com:8083/graphql and http://ipld-eth-server.example.com:8082. More advanced configurations are supported by modifying the watcher's [config file](../../config/watcher-mobymask/mobymask-watcher.toml).
 

stack_orchestrator/data/stacks/monitoring/README.md

@@ -140,6 +140,10 @@ Set the following env variables in the deployment env config file (`monitoring-d
   # External ETH RPC endpoint (filecoin)
   # (Optional, default: https://api.node.glif.io/rpc/v1)
   CERC_FIL_RPC_ENDPOINT=
+
+  # Grafana server host URL (used in various links in alerts, etc.)
+  # (Optional, default: http://localhost:3000)
+  GF_SERVER_ROOT_URL=
   ```
 
 ## Start the stack

stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md

@@ -35,6 +35,8 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-watchers-spec
 
 ## Configure
 
+### Prometheus scrape config
+
 Add the following scrape configs to prometheus config file (`monitoring-watchers-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder:
 
   ```yml
@@ -100,6 +102,27 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
 
 Add scrape config as done above for any additional watcher to add it to the Watchers dashboard.
 
+### Grafana alerts config
+
+Place the pre-configured watcher alerts rules in Grafana provisioning directory:
+
+  ```bash
+  cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
+  ```
+
+Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points
+
+Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/monitoring/grafana/provisioning/alerting/policies.yaml`) with appropriate object-matchers:
+
+  ```yml
+  ...
+    routes:
+      - receiver: SlackNotifier
+      object_matchers:
+        # Add matchers below
+        - ['grafana_folder', '=', 'WatcherAlerts']
+  ```
+
 ### Env
 
 Set the following env variables in the deployment env config file (`monitoring-watchers-deployment/config.env`):
@@ -107,6 +130,10 @@ Set the following env variables in the deployment env config file (`monitoring-w
   ```bash
   # Infura key to be used
   CERC_INFURA_KEY=
+
+  # Grafana server host URL to be used
+  # (Optional, default: http://localhost:3000)
+  GF_SERVER_ROOT_URL=
   ```
 
 ## Start the stack

stack_orchestrator/data/stacks/package-registry/stack.yml

@@ -3,7 +3,7 @@ name: package-registry
 description: "Local Package Registry"
 repos:
   - git.vdb.to/cerc-io/hosting
-  - gitea.com/gitea/act_runner
+  - gitea.com/telackey/act_runner@telackey/entrypoint
 containers:
   - cerc/act-runner
   - cerc/act-runner-task-executor

stack_orchestrator/data/stacks/ping-pub/README.md

@@ -0,0 +1,10 @@
+# ping-pub
+Experimental block explorer for laconic
+
+```
+laconic-so --stack ping-pub setup-repositories
+laconic-so --stack ping-pub build-containers
+laconic-so --stack ping-pub deploy init --output ping-pub-spec.yml --map-ports-to-host localhost-same
+laconic-so --stack ping-pub deploy create --spec-file ping-pub-spec.yml --deployment-dir pp-deployment
+laconic-so deployment --dir pp-deployment start
+```

stack_orchestrator/data/stacks/ping-pub/stack.yml

@@ -0,0 +1,9 @@
+version: "0.1"
+name: ping-pub
+repos:
+  # fork, but only for config & Dockerfile reasonsb
+  - github.com/LaconicNetwork/explorer@laconic
+containers:
+  - cerc/ping-pub
+pods:
+  - ping-pub

stack_orchestrator/data/stacks/snowballtools-base-backend/stack.yml

@@ -0,0 +1,10 @@
+version: "1.0"
+name: snowballtools-base-backend
+description: "snowballtools-base-backend"
+repos:
+  - github.com/snowball-tools/snowballtools-base
+containers:
+  - cerc/webapp-base
+  - cerc/snowballtools-base-backend
+pods:
+  - snowballtools-base-backend

stack_orchestrator/data/stacks/test-database/README.md

@@ -0,0 +1,3 @@
+# Test Database Stack
+
+A stack with a database for test/demo purposes.

stack_orchestrator/data/stacks/test-database/stack.yml

@@ -0,0 +1,9 @@
+version: "1.0"
+name: test
+description: "A test database stack"
+repos:
+containers:
+  - cerc/test-database-container
+  - cerc/test-database-client
+pods:
+  - test-database

stack_orchestrator/data/stacks/webapp-deployer-backend/stack.yml

@@ -0,0 +1,11 @@
+version: "1.0"
+name: webapp-deployer-backend
+description: "Deployer for webapps"
+repos:
+  - git.vdb.to/telackey/webapp-deployment-status-api
+containers:
+  - cerc/webapp-deployer-backend
+pods:
+  - name: webapp-deployer-backend
+    repository: git.vdb.to/telackey/webapp-deployment-status-api
+    path: ./

stack_orchestrator/deploy/compose/deploy_docker.py

@@ -17,6 +17,7 @@ from pathlib import Path
 from python_on_whales import DockerClient, DockerException
 from stack_orchestrator.deploy.deployer import Deployer, DeployerException, DeployerConfigGenerator
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
+from stack_orchestrator.opts import opts
 
 
 class DockerDeployer(Deployer):
@@ -29,60 +30,69 @@ class DockerDeployer(Deployer):
         self.type = type
 
     def up(self, detach, services):
-        try:
-            return self.docker.compose.up(detach=detach, services=services)
-        except DockerException as e:
-            raise DeployerException(e)
+        if not opts.o.dry_run:
+            try:
+                return self.docker.compose.up(detach=detach, services=services)
+            except DockerException as e:
+                raise DeployerException(e)
 
     def down(self, timeout, volumes):
-        try:
-            return self.docker.compose.down(timeout=timeout, volumes=volumes)
-        except DockerException as e:
-            raise DeployerException(e)
+        if not opts.o.dry_run:
+            try:
+                return self.docker.compose.down(timeout=timeout, volumes=volumes)
+            except DockerException as e:
+                raise DeployerException(e)
 
     def update(self):
-        try:
-            return self.docker.compose.restart()
-        except DockerException as e:
-            raise DeployerException(e)
+        if not opts.o.dry_run:
+            try:
+                return self.docker.compose.restart()
+            except DockerException as e:
+                raise DeployerException(e)
 
     def status(self):
-        try:
-            for p in self.docker.compose.ps():
-                print(f"{p.name}\t{p.state.status}")
-        except DockerException as e:
-            raise DeployerException(e)
+        if not opts.o.dry_run:
+            try:
+                for p in self.docker.compose.ps():
+                    print(f"{p.name}\t{p.state.status}")
+            except DockerException as e:
+                raise DeployerException(e)
 
     def ps(self):
-        try:
-            return self.docker.compose.ps()
-        except DockerException as e:
-            raise DeployerException(e)
+        if not opts.o.dry_run:
+            try:
+                return self.docker.compose.ps()
+            except DockerException as e:
+                raise DeployerException(e)
 
     def port(self, service, private_port):
-        try:
-            return self.docker.compose.port(service=service, private_port=private_port)
-        except DockerException as e:
-            raise DeployerException(e)
+        if not opts.o.dry_run:
+            try:
+                return self.docker.compose.port(service=service, private_port=private_port)
+            except DockerException as e:
+                raise DeployerException(e)
 
     def execute(self, service, command, tty, envs):
-        try:
-            return self.docker.compose.execute(service=service, command=command, tty=tty, envs=envs)
-        except DockerException as e:
-            raise DeployerException(e)
+        if not opts.o.dry_run:
+            try:
+                return self.docker.compose.execute(service=service, command=command, tty=tty, envs=envs)
+            except DockerException as e:
+                raise DeployerException(e)
 
     def logs(self, services, tail, follow, stream):
-        try:
-            return self.docker.compose.logs(services=services, tail=tail, follow=follow, stream=stream)
-        except DockerException as e:
-            raise DeployerException(e)
+        if not opts.o.dry_run:
+            try:
+                return self.docker.compose.logs(services=services, tail=tail, follow=follow, stream=stream)
+            except DockerException as e:
+                raise DeployerException(e)
 
     def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False):
-        try:
-            return self.docker.run(image=image, command=command, user=user, volumes=volumes,
-                                   entrypoint=entrypoint, envs=env, detach=detach, publish=ports, publish_all=len(ports) == 0)
-        except DockerException as e:
-            raise DeployerException(e)
+        if not opts.o.dry_run:
+            try:
+                return self.docker.run(image=image, command=command, user=user, volumes=volumes,
+                                       entrypoint=entrypoint, envs=env, detach=detach, publish=ports, publish_all=len(ports) == 0)
+            except DockerException as e:
+                raise DeployerException(e)
 
 
 class DockerDeployerConfigGenerator(DeployerConfigGenerator):

stack_orchestrator/deploy/deploy.py

@@ -85,54 +85,39 @@ def create_deploy_context(
 def up_operation(ctx, services_list, stay_attached=False):
     global_context = ctx.parent.parent.obj
     deploy_context = ctx.obj
-    if not global_context.dry_run:
-        cluster_context = deploy_context.cluster_context
-        container_exec_env = _make_runtime_env(global_context)
-        for attr, value in container_exec_env.items():
-            os.environ[attr] = value
-        if global_context.verbose:
-            print(f"Running compose up with container_exec_env: {container_exec_env}, extra_args: {services_list}")
-        for pre_start_command in cluster_context.pre_start_commands:
-            _run_command(global_context, cluster_context.cluster, pre_start_command)
-        deploy_context.deployer.up(detach=not stay_attached, services=services_list)
-        for post_start_command in cluster_context.post_start_commands:
-            _run_command(global_context, cluster_context.cluster, post_start_command)
-        _orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.deployer, container_exec_env)
+    cluster_context = deploy_context.cluster_context
+    container_exec_env = _make_runtime_env(global_context)
+    for attr, value in container_exec_env.items():
+        os.environ[attr] = value
+    if global_context.verbose:
+        print(f"Running compose up with container_exec_env: {container_exec_env}, extra_args: {services_list}")
+    for pre_start_command in cluster_context.pre_start_commands:
+        _run_command(global_context, cluster_context.cluster, pre_start_command)
+    deploy_context.deployer.up(detach=not stay_attached, services=services_list)
+    for post_start_command in cluster_context.post_start_commands:
+        _run_command(global_context, cluster_context.cluster, post_start_command)
+    _orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.deployer, container_exec_env)
 
 
 def down_operation(ctx, delete_volumes, extra_args_list):
-    global_context = ctx.parent.parent.obj
-    if not global_context.dry_run:
-        if global_context.verbose:
-            print("Running compose down")
-        timeout_arg = None
-        if extra_args_list:
-            timeout_arg = extra_args_list[0]
-        # Specify shutdown timeout (default 10s) to give services enough time to shutdown gracefully
-        ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes)
+    timeout_arg = None
+    if extra_args_list:
+        timeout_arg = extra_args_list[0]
+    # Specify shutdown timeout (default 10s) to give services enough time to shutdown gracefully
+    ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes)
 
 
 def status_operation(ctx):
-    global_context = ctx.parent.parent.obj
-    if not global_context.dry_run:
-        if global_context.verbose:
-            print("Running compose status")
-        ctx.obj.deployer.status()
+    ctx.obj.deployer.status()
 
 
 def update_operation(ctx):
-    global_context = ctx.parent.parent.obj
-    if not global_context.dry_run:
-        if global_context.verbose:
-            print("Running compose update")
-        ctx.obj.deployer.update()
+    ctx.obj.deployer.update()
 
 
 def ps_operation(ctx):
     global_context = ctx.parent.parent.obj
     if not global_context.dry_run:
-        if global_context.verbose:
-            print("Running compose ps")
         container_list = ctx.obj.deployer.ps()
         if len(container_list) > 0:
             print("Running containers:")
@@ -187,15 +172,11 @@ def exec_operation(ctx, extra_args):
 
 
 def logs_operation(ctx, tail: int, follow: bool, extra_args: str):
-    global_context = ctx.parent.parent.obj
     extra_args_list = list(extra_args) or None
-    if not global_context.dry_run:
-        if global_context.verbose:
-            print("Running compose logs")
-        services_list = extra_args_list if extra_args_list is not None else []
-        logs_stream = ctx.obj.deployer.logs(services=services_list, tail=tail, follow=follow, stream=True)
-        for stream_type, stream_content in logs_stream:
-            print(stream_content.decode("utf-8"), end="")
+    services_list = extra_args_list if extra_args_list is not None else []
+    logs_stream = ctx.obj.deployer.logs(services=services_list, tail=tail, follow=follow, stream=True)
+    for stream_type, stream_content in logs_stream:
+        print(stream_content.decode("utf-8"), end="")
 
 
 @command.command()
@@ -347,8 +328,8 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
             else:
                 if deployment:
                     compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml")
-                    pod_pre_start_command = pod["pre_start_command"]
-                    pod_post_start_command = pod["post_start_command"]
+                    pod_pre_start_command = pod.get("pre_start_command")
+                    pod_post_start_command = pod.get("post_start_command")
                     script_dir = compose_dir.parent.joinpath("pods", pod_name, "scripts")
                     if pod_pre_start_command is not None:
                         pre_start_commands.append(os.path.join(script_dir, pod_pre_start_command))
@@ -357,8 +338,8 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
                 else:
                     pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
                     compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml")
-                    pod_pre_start_command = pod["pre_start_command"]
-                    pod_post_start_command = pod["post_start_command"]
+                    pod_pre_start_command = pod.get("pre_start_command")
+                    pod_post_start_command = pod.get("post_start_command")
                     if pod_pre_start_command is not None:
                         pre_start_commands.append(os.path.join(pod_root_dir, pod_pre_start_command))
                     if pod_post_start_command is not None:
@@ -463,7 +444,7 @@ def _orchestrate_cluster_config(ctx, cluster_config, deployer, container_exec_en
                                                               tty=False,
                                                               envs=container_exec_env)
                         waiting_for_data = False
-                    if ctx.debug:
+                    if ctx.debug and not waiting_for_data:
                         print(f"destination output: {destination_output}")
 
 

stack_orchestrator/deploy/deployer_factory.py

@@ -18,11 +18,11 @@ from stack_orchestrator.deploy.k8s.deploy_k8s import K8sDeployer, K8sDeployerCon
 from stack_orchestrator.deploy.compose.deploy_docker import DockerDeployer, DockerDeployerConfigGenerator
 
 
-def getDeployerConfigGenerator(type: str):
+def getDeployerConfigGenerator(type: str, deployment_context):
     if type == "compose" or type is None:
         return DockerDeployerConfigGenerator(type)
     elif type == constants.k8s_deploy_type or type == constants.k8s_kind_deploy_type:
-        return K8sDeployerConfigGenerator(type)
+        return K8sDeployerConfigGenerator(type, deployment_context)
     else:
         print(f"ERROR: deploy-to {type} is not valid")
 

stack_orchestrator/deploy/deployment_create.py

@@ -27,6 +27,7 @@ from stack_orchestrator.opts import opts
 from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
                                      global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
                                      get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file)
+from stack_orchestrator.deploy.spec import Spec
 from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand
 from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
@@ -54,19 +55,44 @@ def _get_ports(stack):
 
 def _get_named_volumes(stack):
     # Parse the compose files looking for named volumes
-    named_volumes = []
+    named_volumes = {
+        "rw": [],
+        "ro": []
+    }
     parsed_stack = get_parsed_stack_config(stack)
     pods = get_pod_list(parsed_stack)
     yaml = get_yaml()
+
+    def find_vol_usage(parsed_pod_file, vol):
+        ret = {}
+        if "services" in parsed_pod_file:
+            for svc_name, svc in parsed_pod_file["services"].items():
+                if "volumes" in svc:
+                    for svc_volume in svc["volumes"]:
+                        parts = svc_volume.split(":")
+                        if parts[0] == vol:
+                            ret[svc_name] = {
+                                "volume": parts[0],
+                                "mount": parts[1],
+                                "options": parts[2] if len(parts) == 3 else None
+                            }
+        return ret
+
     for pod in pods:
         pod_file_path = get_pod_file_path(parsed_stack, pod)
         parsed_pod_file = yaml.load(open(pod_file_path, "r"))
         if "volumes" in parsed_pod_file:
             volumes = parsed_pod_file["volumes"]
             for volume in volumes.keys():
-                # Volume definition looks like:
-                # 'laconicd-data': None
-                named_volumes.append(volume)
+                for vu in find_vol_usage(parsed_pod_file, volume).values():
+                    read_only = vu["options"] == "ro"
+                    if read_only:
+                        if vu["volume"] not in named_volumes["rw"] and vu["volume"] not in named_volumes["ro"]:
+                            named_volumes["ro"].append(vu["volume"])
+                    else:
+                        if vu["volume"] not in named_volumes["rw"]:
+                            named_volumes["rw"].append(vu["volume"])
+
     return named_volumes
 
 
@@ -86,6 +112,7 @@ def _create_bind_dir_if_relative(volume, path_string, compose_dir):
 
 # See: https://stackoverflow.com/questions/45699189/editing-docker-compose-yml-with-pyyaml
 def _fixup_pod_file(pod, spec, compose_dir):
+    deployment_type = spec[constants.deploy_to_key]
     # Fix up volumes
     if "volumes" in spec:
         spec_volumes = spec["volumes"]
@@ -94,16 +121,35 @@ def _fixup_pod_file(pod, spec, compose_dir):
             for volume in pod_volumes.keys():
                 if volume in spec_volumes:
                     volume_spec = spec_volumes[volume]
-                    volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}"
-                    _create_bind_dir_if_relative(volume, volume_spec, compose_dir)
-                    new_volume_spec = {"driver": "local",
-                                       "driver_opts": {
-                                          "type": "none",
-                                          "device": volume_spec_fixedup,
-                                          "o": "bind"
-                                       }
-                                       }
-                    pod["volumes"][volume] = new_volume_spec
+                    if volume_spec:
+                        volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}"
+                        _create_bind_dir_if_relative(volume, volume_spec, compose_dir)
+                        # this is Docker specific
+                        if spec.is_docker_deployment():
+                            new_volume_spec = {
+                                "driver": "local",
+                                "driver_opts": {
+                                    "type": "none",
+                                    "device": volume_spec_fixedup,
+                                    "o": "bind"
+                                }
+                            }
+                            pod["volumes"][volume] = new_volume_spec
+
+    # Fix up configmaps
+    if constants.configmaps_key in spec:
+        if spec.is_kubernetes_deployment():
+            spec_cfgmaps = spec[constants.configmaps_key]
+            if "volumes" in pod:
+                pod_volumes = pod[constants.volumes_key]
+                for volume in pod_volumes.keys():
+                    if volume in spec_cfgmaps:
+                        volume_cfg = spec_cfgmaps[volume]
+                        # Just make the dir (if necessary)
+                        _create_bind_dir_if_relative(volume, volume_cfg, compose_dir)
+        else:
+            print(f"Warning: ConfigMaps not supported for {deployment_type}")
+
     # Fix up ports
     if "network" in spec and "ports" in spec["network"]:
         spec_ports = spec["network"]["ports"]
@@ -286,7 +332,7 @@ def init_operation(deploy_command_context, stack, deployer_type, config,
         if image_registry is None:
             error_exit("--image-registry must be supplied with --deploy-to k8s")
         spec_file_content.update({constants.kube_config_key: kube_config})
-        spec_file_content.update({constants.image_resigtry_key: image_registry})
+        spec_file_content.update({constants.image_registry_key: image_registry})
     else:
         # Check for --kube-config supplied for non-relevant deployer types
         if kube_config is not None:
@@ -319,9 +365,24 @@ def init_operation(deploy_command_context, stack, deployer_type, config,
     named_volumes = _get_named_volumes(stack)
     if named_volumes:
         volume_descriptors = {}
-        for named_volume in named_volumes:
-            volume_descriptors[named_volume] = f"./data/{named_volume}"
-        spec_file_content["volumes"] = volume_descriptors
+        configmap_descriptors = {}
+        for named_volume in named_volumes["rw"]:
+            if "k8s" in deployer_type:
+                volume_descriptors[named_volume] = None
+            else:
+                volume_descriptors[named_volume] = f"./data/{named_volume}"
+        for named_volume in named_volumes["ro"]:
+            if "k8s" in deployer_type:
+                if "config" in named_volume:
+                    configmap_descriptors[named_volume] = f"./configmaps/{named_volume}"
+                else:
+                    volume_descriptors[named_volume] = None
+            else:
+                volume_descriptors[named_volume] = f"./data/{named_volume}"
+        if volume_descriptors:
+            spec_file_content["volumes"] = volume_descriptors
+        if configmap_descriptors:
+            spec_file_content["configmaps"] = configmap_descriptors
 
     if opts.o.debug:
         print(f"Creating spec file for stack: {stack} with content: {spec_file_content}")
@@ -360,6 +421,17 @@ def _create_deployment_file(deployment_dir: Path):
         output_file.write(f"{constants.cluster_id_key}: {cluster}\n")
 
 
+def _check_volume_definitions(spec):
+    if spec.is_kubernetes_deployment():
+        for volume_name, volume_path in spec.get_volumes().items():
+            if volume_path:
+                if not os.path.isabs(volume_path):
+                    raise Exception(
+                        f"Relative path {volume_path} for volume {volume_name} not "
+                        f"supported for deployment type {spec.get_deployment_type()}"
+                    )
+
+
 @click.command()
 @click.option("--spec-file", required=True, help="Spec file to use to create this deployment")
 @click.option("--deployment-dir", help="Create deployment files in this directory")
@@ -375,7 +447,8 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
 # The init command's implementation is in a separate function so that we can
 # call it from other commands, bypassing the click decoration stuff
 def create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers):
-    parsed_spec = get_parsed_deployment_spec(spec_file)
+    parsed_spec = Spec(os.path.abspath(spec_file), get_parsed_deployment_spec(spec_file))
+    _check_volume_definitions(parsed_spec)
     stack_name = parsed_spec["stack"]
     deployment_type = parsed_spec[constants.deploy_to_key]
     stack_file = get_stack_file_path(stack_name)
@@ -441,10 +514,10 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
     deployment_context = DeploymentContext()
     deployment_context.init(deployment_dir_path)
     # Call the deployer to generate any deployer-specific files (e.g. for kind)
-    deployer_config_generator = getDeployerConfigGenerator(deployment_type)
+    deployer_config_generator = getDeployerConfigGenerator(deployment_type, deployment_context)
     # TODO: make deployment_dir_path a Path above
     deployer_config_generator.generate(deployment_dir_path)
-    call_stack_deploy_create(deployment_context, [network_dir, initial_peers])
+    call_stack_deploy_create(deployment_context, [network_dir, initial_peers, deployment_command_context])
 
 
 # TODO: this code should be in the stack .py files but

stack_orchestrator/deploy/images.py

@@ -31,7 +31,8 @@ def _image_needs_pushed(image: str):
 
 def remote_tag_for_image(image: str, remote_repo_url: str):
     # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
-    (org, image_name_with_version) = image.split("/")
+    major_parts = image.split("/", 2)
+    image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
     (image_name, image_version) = image_name_with_version.split(":")
     if image_version == "local":
         return f"{remote_repo_url}/{image_name}:deploy"
@@ -45,7 +46,7 @@ def push_images_operation(command_context: DeployCommandContext, deployment_cont
     cluster_context = command_context.cluster_context
     images: Set[str] = images_for_deployment(cluster_context.compose_files)
     # Tag the images for the remote repo
-    remote_repo_url = deployment_context.spec.obj[constants.image_resigtry_key]
+    remote_repo_url = deployment_context.spec.obj[constants.image_registry_key]
     docker = DockerClient()
     for image in images:
         if _image_needs_pushed(image):

stack_orchestrator/deploy/k8s/cluster_info.py

@@ -13,19 +13,50 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
+import os
+
 from kubernetes import client
 from typing import Any, List, Set
 
 from stack_orchestrator.opts import opts
 from stack_orchestrator.util import env_var_map_from_file
 from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files
-from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path
-from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map
+from stack_orchestrator.deploy.k8s.helpers import get_kind_pv_bind_mount_path
+from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map, envs_from_compose_file, merge_envs
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment
 from stack_orchestrator.deploy.deploy_types import DeployEnvVars
-from stack_orchestrator.deploy.spec import Spec
+from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits
 from stack_orchestrator.deploy.images import remote_tag_for_image
 
+DEFAULT_VOLUME_RESOURCES = Resources({
+    "reservations": {"storage": "2Gi"}
+})
+
+DEFAULT_CONTAINER_RESOURCES = Resources({
+    "reservations": {"cpus": "0.1", "memory": "200M"},
+    "limits": {"cpus": "1.0", "memory": "2000M"},
+})
+
+
+def to_k8s_resource_requirements(resources: Resources) -> client.V1ResourceRequirements:
+    def to_dict(limits: ResourceLimits):
+        if not limits:
+            return None
+
+        ret = {}
+        if limits.cpus:
+            ret["cpu"] = str(limits.cpus)
+        if limits.memory:
+            ret["memory"] = f"{int(limits.memory / (1000 * 1000))}M"
+        if limits.storage:
+            ret["storage"] = f"{int(limits.storage / (1000 * 1000))}M"
+        return ret
+
+    return client.V1ResourceRequirements(
+        requests=to_dict(resources.reservations),
+        limits=to_dict(resources.limits)
+    )
+
 
 class ClusterInfo:
     parsed_pod_yaml_map: Any
@@ -47,7 +78,30 @@ class ClusterInfo:
         if (opts.o.debug):
             print(f"Env vars: {self.environment_variables.map}")
 
-    def get_ingress(self):
+    def get_nodeport(self):
+        for pod_name in self.parsed_pod_yaml_map:
+            pod = self.parsed_pod_yaml_map[pod_name]
+            services = pod["services"]
+            for service_name in services:
+                service_info = services[service_name]
+                if "ports" in service_info:
+                    port = int(service_info["ports"][0])
+                    if opts.o.debug:
+                        print(f"service port: {port}")
+        service = client.V1Service(
+            metadata=client.V1ObjectMeta(name=f"{self.app_name}-nodeport"),
+            spec=client.V1ServiceSpec(
+                type="NodePort",
+                ports=[client.V1ServicePort(
+                    port=port,
+                    target_port=port
+                )],
+                selector={"app": self.app_name}
+            )
+        )
+        return service
+
+    def get_ingress(self, use_tls=False):
         # No ingress for a deployment that has no http-proxy defined, for now
         http_proxy_info_list = self.spec.get_http_proxy()
         ingress = None
@@ -62,7 +116,7 @@ class ClusterInfo:
             tls = [client.V1IngressTLS(
                 hosts=[host_name],
                 secret_name=f"{self.app_name}-tls"
-            )]
+            )] if use_tls else None
             paths = []
             for route in http_proxy_info["routes"]:
                 path = route["path"]
@@ -112,9 +166,10 @@ class ClusterInfo:
             services = pod["services"]
             for service_name in services:
                 service_info = services[service_name]
-                port = int(service_info["ports"][0])
-                if opts.o.debug:
-                    print(f"service port: {port}")
+                if "ports" in service_info:
+                    port = int(service_info["ports"][0])
+                    if opts.o.debug:
+                        print(f"service port: {port}")
         service = client.V1Service(
             metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"),
             spec=client.V1ServiceSpec(
@@ -130,47 +185,123 @@ class ClusterInfo:
 
     def get_pvcs(self):
         result = []
-        volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        spec_volumes = self.spec.get_volumes()
+        named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        resources = self.spec.get_volume_resources()
+        if not resources:
+            resources = DEFAULT_VOLUME_RESOURCES
         if opts.o.debug:
-            print(f"Volumes: {volumes}")
-        for volume_name in volumes:
+            print(f"Spec Volumes: {spec_volumes}")
+            print(f"Named Volumes: {named_volumes}")
+            print(f"Resources: {resources}")
+        for volume_name, volume_path in spec_volumes.items():
+            if volume_name not in named_volumes:
+                if opts.o.debug:
+                    print(f"{volume_name} not in pod files")
+                continue
+
+            labels = {
+                "app": self.app_name,
+                "volume-label": f"{self.app_name}-{volume_name}"
+            }
+            if volume_path:
+                storage_class_name = "manual"
+                k8s_volume_name = f"{self.app_name}-{volume_name}"
+            else:
+                # These will be auto-assigned.
+                storage_class_name = None
+                k8s_volume_name = None
+
             spec = client.V1PersistentVolumeClaimSpec(
                 access_modes=["ReadWriteOnce"],
-                storage_class_name="manual",
-                resources=client.V1ResourceRequirements(
-                    requests={"storage": "2Gi"}
-                ),
-                volume_name=volume_name
+                storage_class_name=storage_class_name,
+                resources=to_k8s_resource_requirements(resources),
+                volume_name=k8s_volume_name
             )
             pvc = client.V1PersistentVolumeClaim(
-                metadata=client.V1ObjectMeta(name=volume_name,
-                                             labels={"volume-label": volume_name}),
-                spec=spec,
+                metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}", labels=labels),
+                spec=spec
             )
             result.append(pvc)
         return result
 
+    def get_configmaps(self):
+        result = []
+        spec_configmaps = self.spec.get_configmaps()
+        named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        for cfg_map_name, cfg_map_path in spec_configmaps.items():
+            if cfg_map_name not in named_volumes:
+                if opts.o.debug:
+                    print(f"{cfg_map_name} not in pod files")
+                continue
+
+            if not cfg_map_path.startswith("/"):
+                cfg_map_path = os.path.join(os.path.dirname(self.spec.file_path), cfg_map_path)
+
+            # Read in all the files at a single-level of the directory.  This mimics the behavior
+            # of `kubectl create configmap foo --from-file=/path/to/dir`
+            data = {}
+            for f in os.listdir(cfg_map_path):
+                full_path = os.path.join(cfg_map_path, f)
+                if os.path.isfile(full_path):
+                    data[f] = open(full_path, 'rt').read()
+
+            spec = client.V1ConfigMap(
+                metadata=client.V1ObjectMeta(name=f"{self.app_name}-{cfg_map_name}",
+                                             labels={"configmap-label": cfg_map_name}),
+                data=data
+            )
+            result.append(spec)
+        return result
+
     def get_pvs(self):
         result = []
-        volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
-        for volume_name in volumes:
+        spec_volumes = self.spec.get_volumes()
+        named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        resources = self.spec.get_volume_resources()
+        if not resources:
+            resources = DEFAULT_VOLUME_RESOURCES
+        for volume_name, volume_path in spec_volumes.items():
+            # We only need to create a volume if it is fully qualified HostPath.
+            # Otherwise, we create the PVC and expect the node to allocate the volume for us.
+            if not volume_path:
+                if opts.o.debug:
+                    print(f"{volume_name} does not require an explicit PersistentVolume, since it is not a bind-mount.")
+                continue
+
+            if volume_name not in named_volumes:
+                if opts.o.debug:
+                    print(f"{volume_name} not in pod files")
+                continue
+
+            if not os.path.isabs(volume_path):
+                print(f"WARNING: {volume_name}:{volume_path} is not absolute, cannot bind volume.")
+                continue
+
+            if self.spec.is_kind_deployment():
+                host_path = client.V1HostPathVolumeSource(path=get_kind_pv_bind_mount_path(volume_name))
+            else:
+                host_path = client.V1HostPathVolumeSource(path=volume_path)
             spec = client.V1PersistentVolumeSpec(
                 storage_class_name="manual",
                 access_modes=["ReadWriteOnce"],
-                capacity={"storage": "2Gi"},
-                host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name))
+                capacity=to_k8s_resource_requirements(resources).requests,
+                host_path=host_path
             )
             pv = client.V1PersistentVolume(
-                metadata=client.V1ObjectMeta(name=volume_name,
-                                             labels={"volume-label": volume_name}),
+                metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}",
+                                             labels={"volume-label": f"{self.app_name}-{volume_name}"}),
                 spec=spec,
             )
             result.append(pv)
         return result
 
-    # to suit the deployment, and also annotate the container specs to point at said volumes
-    def get_deployment(self):
+    # TODO: put things like image pull policy into an object-scope struct
+    def get_deployment(self, image_pull_policy: str = None):
         containers = []
+        resources = self.spec.get_container_resources()
+        if not resources:
+            resources = DEFAULT_CONTAINER_RESOURCES
         for pod_name in self.parsed_pod_yaml_map:
             pod = self.parsed_pod_yaml_map[pod_name]
             services = pod["services"]
@@ -178,10 +309,18 @@ class ClusterInfo:
                 container_name = service_name
                 service_info = services[service_name]
                 image = service_info["image"]
-                port = int(service_info["ports"][0])
+                if "ports" in service_info:
+                    port = int(service_info["ports"][0])
+                    if opts.o.debug:
+                        print(f"image: {image}")
+                        print(f"service port: {port}")
+                merged_envs = merge_envs(
+                    envs_from_compose_file(
+                        service_info["environment"]), self.environment_variables.map
+                ) if "environment" in service_info else self.environment_variables.map
+                envs = envs_from_environment_variables_map(merged_envs)
                 if opts.o.debug:
-                    print(f"image: {image}")
-                    print(f"service port: {port}")
+                    print(f"Merged envs: {envs}")
                 # Re-write the image tag for remote deployment
                 image_to_use = remote_tag_for_image(
                     image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
@@ -189,20 +328,41 @@ class ClusterInfo:
                 container = client.V1Container(
                     name=container_name,
                     image=image_to_use,
-                    image_pull_policy="Always",
-                    env=envs_from_environment_variables_map(self.environment_variables.map),
+                    image_pull_policy=image_pull_policy,
+                    env=envs,
                     ports=[client.V1ContainerPort(container_port=port)],
                     volume_mounts=volume_mounts,
-                    resources=client.V1ResourceRequirements(
-                        requests={"cpu": "100m", "memory": "200Mi"},
-                        limits={"cpu": "500m", "memory": "500Mi"},
+                    security_context=client.V1SecurityContext(
+                        privileged=self.spec.get_privileged(),
+                        capabilities=client.V1Capabilities(
+                            add=self.spec.get_capabilities()
+                        ) if self.spec.get_capabilities() else None
                     ),
+                    resources=to_k8s_resource_requirements(resources),
                 )
                 containers.append(container)
-        volumes = volumes_for_pod_files(self.parsed_pod_yaml_map)
+        volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec, self.app_name)
         image_pull_secrets = [client.V1LocalObjectReference(name="laconic-registry")]
+
+        annotations = None
+        labels = {"app": self.app_name}
+
+        if self.spec.get_annotations():
+            annotations = {}
+            for key, value in self.spec.get_annotations().items():
+                for service_name in services:
+                    annotations[key.replace("{name}", service_name)] = value
+
+        if self.spec.get_labels():
+            for key, value in self.spec.get_labels().items():
+                for service_name in services:
+                    labels[key.replace("{name}", service_name)] = value
+
         template = client.V1PodTemplateSpec(
-            metadata=client.V1ObjectMeta(labels={"app": self.app_name}),
+            metadata=client.V1ObjectMeta(
+                annotations=annotations,
+                labels=labels
+            ),
             spec=client.V1PodSpec(containers=containers, image_pull_secrets=image_pull_secrets, volumes=volumes),
         )
         spec = client.V1DeploymentSpec(

stack_orchestrator/deploy/k8s/deploy_k8s.py

@@ -1,5 +1,4 @@
 # Copyright © 2023 Vulcanize
-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
@@ -21,7 +20,9 @@ from kubernetes import client, config
 from stack_orchestrator import constants
 from stack_orchestrator.deploy.deployer import Deployer, DeployerConfigGenerator
 from stack_orchestrator.deploy.k8s.helpers import create_cluster, destroy_cluster, load_images_into_kind
-from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, log_stream_from_string, generate_kind_config
+from stack_orchestrator.deploy.k8s.helpers import install_ingress_for_kind, wait_for_ingress_in_kind
+from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, containers_in_pod, log_stream_from_string
+from stack_orchestrator.deploy.k8s.helpers import generate_kind_config
 from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo
 from stack_orchestrator.opts import opts
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
@@ -82,99 +83,190 @@ class K8sDeployer(Deployer):
         self.apps_api = client.AppsV1Api()
         self.custom_obj_api = client.CustomObjectsApi()
 
-    def up(self, detach, services):
-
-        if self.is_kind():
-            # Create the kind cluster
-            create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath(constants.kind_config_filename))
-            # Ensure the referenced containers are copied into kind
-            load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set)
-        self.connect_api()
-
+    def _create_volume_data(self):
         # Create the host-path-mounted PVs for this deployment
         pvs = self.cluster_info.get_pvs()
         for pv in pvs:
             if opts.o.debug:
                 print(f"Sending this pv: {pv}")
-            pv_resp = self.core_api.create_persistent_volume(body=pv)
-            if opts.o.debug:
-                print("PVs created:")
-                print(f"{pv_resp}")
+            if not opts.o.dry_run:
+                try:
+                    pv_resp = self.core_api.read_persistent_volume(name=pv.metadata.name)
+                    if pv_resp:
+                        if opts.o.debug:
+                            print("PVs already present:")
+                            print(f"{pv_resp}")
+                        continue
+                except:  # noqa: E722
+                    pass
+
+                pv_resp = self.core_api.create_persistent_volume(body=pv)
+                if opts.o.debug:
+                    print("PVs created:")
+                    print(f"{pv_resp}")
 
         # Figure out the PVCs for this deployment
         pvcs = self.cluster_info.get_pvcs()
         for pvc in pvcs:
             if opts.o.debug:
                 print(f"Sending this pvc: {pvc}")
-            pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace=self.k8s_namespace)
+
+            if not opts.o.dry_run:
+                try:
+                    pvc_resp = self.core_api.read_namespaced_persistent_volume_claim(
+                        name=pvc.metadata.name, namespace=self.k8s_namespace)
+                    if pvc_resp:
+                        if opts.o.debug:
+                            print("PVCs already present:")
+                            print(f"{pvc_resp}")
+                        continue
+                except:  # noqa: E722
+                    pass
+
+                pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace=self.k8s_namespace)
+                if opts.o.debug:
+                    print("PVCs created:")
+                    print(f"{pvc_resp}")
+
+        # Figure out the ConfigMaps for this deployment
+        config_maps = self.cluster_info.get_configmaps()
+        for cfg_map in config_maps:
             if opts.o.debug:
-                print("PVCs created:")
-                print(f"{pvc_resp}")
+                print(f"Sending this ConfigMap: {cfg_map}")
+            if not opts.o.dry_run:
+                cfg_rsp = self.core_api.create_namespaced_config_map(
+                    body=cfg_map,
+                    namespace=self.k8s_namespace
+                )
+                if opts.o.debug:
+                    print("ConfigMap created:")
+                    print(f"{cfg_rsp}")
+
+    def _create_deployment(self):
         # Process compose files into a Deployment
-        deployment = self.cluster_info.get_deployment()
+        deployment = self.cluster_info.get_deployment(image_pull_policy=None if self.is_kind() else "Always")
         # Create the k8s objects
         if opts.o.debug:
             print(f"Sending this deployment: {deployment}")
-        deployment_resp = self.apps_api.create_namespaced_deployment(
-            body=deployment, namespace=self.k8s_namespace
-        )
-        if opts.o.debug:
-            print("Deployment created:")
-            print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \
-                  {deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}")
+        if not opts.o.dry_run:
+            deployment_resp = self.apps_api.create_namespaced_deployment(
+                body=deployment, namespace=self.k8s_namespace
+            )
+            if opts.o.debug:
+                print("Deployment created:")
+                print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \
+                    {deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}")
 
         service: client.V1Service = self.cluster_info.get_service()
-        service_resp = self.core_api.create_namespaced_service(
-            namespace=self.k8s_namespace,
-            body=service
-        )
         if opts.o.debug:
-            print("Service created:")
-            print(f"{service_resp}")
-
-        # TODO: disable ingress for kind
-        ingress: client.V1Ingress = self.cluster_info.get_ingress()
-
-        if opts.o.debug:
-            print(f"Sending this ingress: {ingress}")
-        ingress_resp = self.networking_api.create_namespaced_ingress(
-            namespace=self.k8s_namespace,
-            body=ingress
-        )
-        if opts.o.debug:
-            print("Ingress created:")
-            print(f"{ingress_resp}")
-
-    def down(self, timeout, volumes):
-        self.connect_api()
-        # Delete the k8s objects
-        # Create the host-path-mounted PVs for this deployment
-        pvs = self.cluster_info.get_pvs()
-        for pv in pvs:
+            print(f"Sending this service: {service}")
+        if not opts.o.dry_run:
+            service_resp = self.core_api.create_namespaced_service(
+                namespace=self.k8s_namespace,
+                body=service
+            )
             if opts.o.debug:
-                print(f"Deleting this pv: {pv}")
-            try:
-                pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name)
-                if opts.o.debug:
-                    print("PV deleted:")
-                    print(f"{pv_resp}")
-            except client.exceptions.ApiException as e:
-                _check_delete_exception(e)
+                print("Service created:")
+                print(f"{service_resp}")
 
-        # Figure out the PVCs for this deployment
-        pvcs = self.cluster_info.get_pvcs()
-        for pvc in pvcs:
+    def up(self, detach, services):
+        if not opts.o.dry_run:
+            if self.is_kind():
+                # Create the kind cluster
+                create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath(constants.kind_config_filename))
+                # Ensure the referenced containers are copied into kind
+                load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set)
+            self.connect_api()
+            if self.is_kind():
+                # Now configure an ingress controller (not installed by default in kind)
+                install_ingress_for_kind()
+                # Wait for ingress to start (deployment provisioning will fail unless this is done)
+                wait_for_ingress_in_kind()
+
+        else:
+            print("Dry run mode enabled, skipping k8s API connect")
+
+        self._create_volume_data()
+        self._create_deployment()
+
+        # Note: at present we don't support tls for kind (and enabling tls causes errors)
+        ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=not self.is_kind())
+        if ingress:
             if opts.o.debug:
-                print(f"Deleting this pvc: {pvc}")
-            try:
-                pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim(
-                    name=pvc.metadata.name, namespace=self.k8s_namespace
+                print(f"Sending this ingress: {ingress}")
+            if not opts.o.dry_run:
+                ingress_resp = self.networking_api.create_namespaced_ingress(
+                    namespace=self.k8s_namespace,
+                    body=ingress
                 )
                 if opts.o.debug:
-                    print("PVCs deleted:")
-                    print(f"{pvc_resp}")
+                    print("Ingress created:")
+                    print(f"{ingress_resp}")
+        else:
+            if opts.o.debug:
+                print("No ingress configured")
+
+        nodeport: client.V1Service = self.cluster_info.get_nodeport()
+        if nodeport:
+            if opts.o.debug:
+                print(f"Sending this nodeport: {nodeport}")
+            if not opts.o.dry_run:
+                nodeport_resp = self.core_api.create_namespaced_service(
+                    namespace=self.k8s_namespace,
+                    body=nodeport
+                )
+                if opts.o.debug:
+                    print("NodePort created:")
+                    print(f"{nodeport_resp}")
+
+    def down(self, timeout, volumes):  # noqa: C901
+        self.connect_api()
+        # Delete the k8s objects
+
+        if volumes:
+            # Create the host-path-mounted PVs for this deployment
+            pvs = self.cluster_info.get_pvs()
+            for pv in pvs:
+                if opts.o.debug:
+                    print(f"Deleting this pv: {pv}")
+                try:
+                    pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name)
+                    if opts.o.debug:
+                        print("PV deleted:")
+                        print(f"{pv_resp}")
+                except client.exceptions.ApiException as e:
+                    _check_delete_exception(e)
+
+            # Figure out the PVCs for this deployment
+            pvcs = self.cluster_info.get_pvcs()
+            for pvc in pvcs:
+                if opts.o.debug:
+                    print(f"Deleting this pvc: {pvc}")
+                try:
+                    pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim(
+                        name=pvc.metadata.name, namespace=self.k8s_namespace
+                    )
+                    if opts.o.debug:
+                        print("PVCs deleted:")
+                        print(f"{pvc_resp}")
+                except client.exceptions.ApiException as e:
+                    _check_delete_exception(e)
+
+        # Figure out the ConfigMaps for this deployment
+        cfg_maps = self.cluster_info.get_configmaps()
+        for cfg_map in cfg_maps:
+            if opts.o.debug:
+                print(f"Deleting this ConfigMap: {cfg_map}")
+            try:
+                cfg_map_resp = self.core_api.delete_namespaced_config_map(
+                    name=cfg_map.metadata.name, namespace=self.k8s_namespace
+                )
+                if opts.o.debug:
+                    print("ConfigMap deleted:")
+                    print(f"{cfg_map_resp}")
             except client.exceptions.ApiException as e:
                 _check_delete_exception(e)
+
         deployment = self.cluster_info.get_deployment()
         if opts.o.debug:
             print(f"Deleting this deployment: {deployment}")
@@ -196,16 +288,34 @@ class K8sDeployer(Deployer):
         except client.exceptions.ApiException as e:
             _check_delete_exception(e)
 
-        # TODO: disable ingress for kind
-        ingress: client.V1Ingress = self.cluster_info.get_ingress()
-        if opts.o.debug:
-            print(f"Deleting this ingress: {ingress}")
-        try:
-            self.networking_api.delete_namespaced_ingress(
-                name=ingress.metadata.name, namespace=self.k8s_namespace
-            )
-        except client.exceptions.ApiException as e:
-            _check_delete_exception(e)
+        ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=not self.is_kind())
+        if ingress:
+            if opts.o.debug:
+                print(f"Deleting this ingress: {ingress}")
+            try:
+                self.networking_api.delete_namespaced_ingress(
+                    name=ingress.metadata.name, namespace=self.k8s_namespace
+                )
+            except client.exceptions.ApiException as e:
+                _check_delete_exception(e)
+        else:
+            if opts.o.debug:
+                print("No ingress to delete")
+
+        nodeport: client.V1Service = self.cluster_info.get_nodeport()
+        if nodeport:
+            if opts.o.debug:
+                print(f"Deleting this nodeport: {ingress}")
+            try:
+                self.core_api.delete_namespaced_service(
+                    namespace=self.k8s_namespace,
+                    name=nodeport.metadata.name
+                )
+            except client.exceptions.ApiException as e:
+                _check_delete_exception(e)
+        else:
+            if opts.o.debug:
+                print("No nodeport to delete")
 
         if self.is_kind():
             # Destroy the kind cluster
@@ -219,7 +329,7 @@ class K8sDeployer(Deployer):
 
         if all_pods.items:
             for p in all_pods.items:
-                if self.cluster_info.app_name in p.metadata.name:
+                if f"{self.cluster_info.app_name}-deployment" in p.metadata.name:
                     pods.append(p)
 
         if not pods:
@@ -266,7 +376,7 @@ class K8sDeployer(Deployer):
         ret = []
 
         for p in pods.items:
-            if self.cluster_info.app_name in p.metadata.name:
+            if f"{self.cluster_info.app_name}-deployment" in p.metadata.name:
                 pod_ip = p.status.pod_ip
                 ports = AttrDict()
                 for c in p.spec.containers:
@@ -299,11 +409,26 @@ class K8sDeployer(Deployer):
 
     def logs(self, services, tail, follow, stream):
         self.connect_api()
-        pods = pods_in_deployment(self.core_api, "test-deployment")
+        pods = pods_in_deployment(self.core_api, self.cluster_info.app_name)
         if len(pods) > 1:
             print("Warning: more than one pod in the deployment")
-        k8s_pod_name = pods[0]
-        log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
+        if len(pods) == 0:
+            log_data = "******* Pods not running ********\n"
+        else:
+            k8s_pod_name = pods[0]
+            containers = containers_in_pod(self.core_api, k8s_pod_name)
+            # If the pod is not yet started, the logs request below will throw an exception
+            try:
+                log_data = ""
+                for container in containers:
+                    container_log = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container=container)
+                    container_log_lines = container_log.splitlines()
+                    for line in container_log_lines:
+                        log_data += f"{container}: {line}\n"
+            except client.exceptions.ApiException as e:
+                if opts.o.debug:
+                    print(f"Error from read_namespaced_pod_log: {e}")
+                log_data = "******* No logs available ********\n"
         return log_stream_from_string(log_data)
 
     def update(self):
@@ -344,8 +469,9 @@ class K8sDeployer(Deployer):
 class K8sDeployerConfigGenerator(DeployerConfigGenerator):
     type: str
 
-    def __init__(self, type: str) -> None:
+    def __init__(self, type: str, deployment_context) -> None:
         self.type = type
+        self.deployment_context = deployment_context
         super().__init__()
 
     def generate(self, deployment_dir: Path):
@@ -353,7 +479,7 @@ class K8sDeployerConfigGenerator(DeployerConfigGenerator):
         if self.type == "k8s-kind":
             # Check the file isn't already there
             # Get the config file contents
-            content = generate_kind_config(deployment_dir)
+            content = generate_kind_config(deployment_dir, self.deployment_context)
             if opts.o.debug:
                 print(f"kind config is: {content}")
             config_file = deployment_dir.joinpath(constants.kind_config_filename)