Add support for running jobs from a stack (#975)

Part of https://plan.wireit.in/deepstack/browse/VUL-265/

Reviewed-on: #975
Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>

.gitea/workflows/fixturenet-eth-plugeth-test.yml

@@ -1,57 +0,0 @@
-name: Fixturenet-Eth-Plugeth-Test
-
-on:
-  push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
-  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
-    - cron: '2 14 * * *'
-
-
-jobs:
-  test:
-    name: "Run an Ethereum plugeth fixturenet test"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: "Run fixturenet-eth tests"
-        run: ./tests/fixturenet-eth-plugeth/run-test.sh
-      - name: Notify Vulcanize Slack on CI failure
-        if: ${{ always() && github.ref_name == 'main' }}
-        uses: ravsamhq/notify-slack-action@v2
-        with:
-          status: ${{ job.status }}
-          notify_when: 'failure'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
-      - name: Notify DeepStack Slack on CI failure
-        if: ${{ always() && github.ref_name == 'main' }}
-        uses: ravsamhq/notify-slack-action@v2
-        with:
-          status: ${{ job.status }}
-          notify_when: 'failure'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/fixturenet-eth-test.yml

@@ -1,55 +0,0 @@
-name: Fixturenet-Eth-Test
-
-on:
-  push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.gitea/workflows/triggers/fixturenet-eth-test'
-
-
-jobs:
-  test:
-    name: "Run an Ethereum fixturenet test"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: "Run fixturenet-eth tests"
-        run: ./tests/fixturenet-eth/run-test.sh
-      - name: Notify Vulcanize Slack on CI failure
-        if: ${{ always() && github.ref_name == 'main' }}
-        uses: ravsamhq/notify-slack-action@v2
-        with:
-          status: ${{ job.status }}
-          notify_when: 'failure'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
-      - name: Notify DeepStack Slack on CI failure
-        if: ${{ always() && github.ref_name == 'main' }}
-        uses: ravsamhq/notify-slack-action@v2
-        with:
-          status: ${{ job.status }}
-          notify_when: 'failure'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -39,7 +39,7 @@ jobs:
       - name: "Print Python version"
         run: python3 --version
       - name: "Install shiv"
-        run: pip install shiv
+        run: pip install shiv==1.0.6
       - name: "Generate build version file"
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"

.gitea/workflows/publish.yml

@@ -35,7 +35,7 @@ jobs:
       - name: "Print Python version"
         run: python3 --version
       - name: "Install shiv"
-        run: pip install shiv
+        run: pip install shiv==1.0.6
       - name: "Build local shiv package"
         id: build
         run: |

.gitea/workflows/test-container-registry.yml

@@ -33,7 +33,7 @@ jobs:
       - name: "Print Python version"
         run: python3 --version
       - name: "Install shiv"
-        run: pip install shiv
+        run: pip install shiv==1.0.6
       - name: "Generate build version file"
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"

.gitea/workflows/test-database.yml

@@ -33,7 +33,7 @@ jobs:
       - name: "Print Python version"
         run: python3 --version
       - name: "Install shiv"
-        run: pip install shiv
+        run: pip install shiv==1.0.6
       - name: "Generate build version file"
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"

.gitea/workflows/test-deploy.yml

@@ -33,7 +33,7 @@ jobs:
       - name: "Print Python version"
         run: python3 --version
       - name: "Install shiv"
-        run: pip install shiv
+        run: pip install shiv==1.0.6
       - name: "Generate build version file"
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"

.gitea/workflows/test-external-stack.yml

@@ -33,7 +33,7 @@ jobs:
       - name: "Print Python version"
         run: python3 --version
       - name: "Install shiv"
-        run: pip install shiv
+        run: pip install shiv==1.0.6
       - name: "Generate build version file"
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"

.gitea/workflows/test-k8s-deploy.yml

@@ -35,7 +35,7 @@ jobs:
       - name: "Print Python version"
         run: python3 --version
       - name: "Install shiv"
-        run: pip install shiv
+        run: pip install shiv==1.0.6
       - name: "Generate build version file"
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"

.gitea/workflows/test-k8s-deployment-control.yml

@@ -1,19 +1,22 @@
-name: Fixturenet-Eth-Plugeth-Arm-Test
+name: K8s Deployment Control Test
 
 on:
+  pull_request:
+    branches: '*'
   push:
     branches: '*'
     paths:
       - '!**'
-      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-arm-test'
+      - '.gitea/workflows/triggers/test-k8s-deployment-control'
+      - '.gitea/workflows/test-k8s-deployment-control.yml'
+      - 'tests/k8s-deployment-control/run-test.sh'
   schedule: # Note: coordinate with other tests to not overload runners at the same time of day
-    - cron: '2 14 * * *'
-
+    - cron: '3 30 * * *'
 
 jobs:
   test:
-    name: "Run an Ethereum plugeth fixturenet test"
-    runs-on: ubuntu-latest-arm
+    name: "Run deployment control suite on kind/k8s"
+    runs-on: ubuntu-22.04
     steps:
       - name: "Clone project repository"
         uses: actions/checkout@v3
@@ -32,13 +35,22 @@ jobs:
       - name: "Print Python version"
         run: python3 --version
       - name: "Install shiv"
-        run: pip install shiv
+        run: pip install shiv==1.0.6
       - name: "Generate build version file"
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: "Run fixturenet-eth tests"
-        run: ./tests/fixturenet-eth-plugeth/run-test.sh
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Run k8s deployment control test"
+        run: |
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/k8s-deployment-control/run-test.sh
       - name: Notify Vulcanize Slack on CI failure
         if: ${{ always() && github.ref_name == 'main' }}
         uses: ravsamhq/notify-slack-action@v2

.gitea/workflows/test-webapp.yml

@@ -32,7 +32,7 @@ jobs:
       - name: "Print Python version"
         run: python3 --version
       - name: "Install shiv"
-        run: pip install shiv
+        run: pip install shiv==1.0.6
       - name: "Generate build version file"
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"

.gitea/workflows/test.yml

@@ -33,7 +33,7 @@ jobs:
       - name: "Print Python version"
         run: python3 --version
       - name: "Install shiv"
-        run: pip install shiv
+        run: pip install shiv==1.0.6
       - name: "Generate build version file"
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"

.gitea/workflows/triggers/fixturenet-eth-plugeth-arm-test

@@ -1,2 +0,0 @@
-Change this file to trigger running the fixturenet-eth-plugeth-arm-test CI job
-

.gitea/workflows/triggers/fixturenet-eth-plugeth-test

@@ -1,3 +0,0 @@
-Change this file to trigger running the fixturenet-eth-plugeth-test CI job
-trigger
-trigger

.gitea/workflows/triggers/fixturenet-eth-test

@@ -1,2 +0,0 @@
-Change this file to trigger running the fixturenet-eth-test CI job
-

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -4,3 +4,7 @@ Trigger
 Trigger
 Trigger
 Trigger
+Trigger
+Trigger
+Trigger
+Trigger

docs/cli.md

@@ -51,7 +51,7 @@ $ laconic-so build-npms --include <package-name>
 ```
 e.g.
 ```
-$ laconic-so build-npms --include laconic-sdk
+$ laconic-so build-npms --include registry-sdk
 ```
 Build the packages for a stack:
 ```

docs/gitea-with-laconicd-fixturenet.md

@@ -56,7 +56,7 @@ laconic-so --stack fixturenet-laconicd build-npms
 Navigate to the Gitea console and switch to the `cerc-io` user then find the `Packages` tab to confirm that these two npm packages have been published:
 
 - `@cerc-io/laconic-registry-cli`
-- `@cerc-io/laconic-sdk`
+- `@cerc-io/registry-sdk`
 
 ### Build and deploy fixturenet containers
 
@@ -74,7 +74,7 @@ laconic-so --stack fixturenet-laconicd deploy logs
 ### Test with the registry CLI
 
 ```bash
-laconic-so --stack fixturenet-laconicd deploy exec cli "laconic cns status"
+laconic-so --stack fixturenet-laconicd deploy exec cli "laconic registry status"
 ```
 
 Try additional CLI commands, documented [here](https://github.com/cerc-io/laconic-registry-cli#operations).

docs/helm-chart-generation.md

@@ -0,0 +1,113 @@
+# Helm Chart Generation
+
+Generate Kubernetes Helm charts from stack compose files using Kompose.
+
+## Prerequisites
+
+Install Kompose:
+
+```bash
+# Linux
+curl -L https://github.com/kubernetes/kompose/releases/download/v1.34.0/kompose-linux-amd64 -o kompose
+chmod +x kompose
+sudo mv kompose /usr/local/bin/
+
+# macOS
+brew install kompose
+
+# Verify
+kompose version
+```
+
+## Usage
+
+### 1. Create spec file
+
+```bash
+laconic-so --stack <stack-name> deploy --deploy-to k8s init \
+  --kube-config ~/.kube/config \
+  --output spec.yml
+```
+
+### 2. Generate Helm chart
+
+```bash
+laconic-so --stack <stack-name> deploy create \
+  --spec-file spec.yml \
+  --deployment-dir my-deployment \
+  --helm-chart
+```
+
+### 3. Deploy to Kubernetes
+
+```bash
+helm install my-release my-deployment/chart
+kubectl get pods -n zenith
+```
+
+## Output Structure
+
+```bash
+my-deployment/
+├── spec.yml              # Reference
+├── stack.yml             # Reference
+└── chart/                # Helm chart
+  ├── Chart.yaml
+  ├── README.md
+  └── templates/
+    └── *.yaml
+```
+
+## Example
+
+```bash
+# Generate chart for stage1-zenithd
+laconic-so --stack stage1-zenithd deploy --deploy-to k8s init \
+  --kube-config ~/.kube/config \
+  --output stage1-spec.yml
+
+laconic-so --stack stage1-zenithd deploy create \
+  --spec-file stage1-spec.yml \
+  --deployment-dir stage1-deployment \
+  --helm-chart
+
+# Deploy
+helm install stage1-zenithd stage1-deployment/chart
+```
+
+## Production Deployment (TODO)
+
+### Local Development
+
+```bash
+# Access services using port-forward
+kubectl port-forward service/zenithd 26657:26657
+kubectl port-forward service/nginx-api-proxy 1317:80
+kubectl port-forward service/cosmos-explorer 4173:4173
+```
+
+### Production Access Options
+
+- Option 1: Ingress + cert-manager (Recommended)
+  - Install ingress-nginx + cert-manager
+  - Point DNS to cluster LoadBalancer IP
+  - Auto-provisions Let's Encrypt TLS certs
+  - Access: `https://api.zenith.example.com`
+- Option 2: Cloud LoadBalancer
+  - Use cloud provider's LoadBalancer service type
+  - Point DNS to assigned external IP
+  - Manual TLS cert management
+- Option 3: Bare Metal (MetalLB + Ingress)
+  - MetalLB provides LoadBalancer IPs from local network
+  - Same Ingress setup as cloud
+- Option 4: NodePort + External Proxy
+  - Expose services on 30000-32767 range
+  - External nginx/Caddy proxies 80/443 → NodePort
+  - Manual cert management
+
+### Changes Needed
+
+- Add Ingress template to charts
+- Add TLS configuration to values.yaml
+- Document cert-manager setup
+- Add production deployment guide

docs/k8s-deployment-enhancements.md

@@ -0,0 +1,27 @@
+# K8S Deployment Enhancements
+## Controlling pod placement
+The placement of pods created as part of a stack deployment can be controlled to either avoid certain nodes, or require certain nodes.
+### Pod/Node Affinity
+Node affinity rules applied to pods target node labels. The effect is that a pod can only be placed on a node having the specified label value. Note that other pods that do not have any node affinity rules can also be placed on those same nodes. Thus node affinity for a pod controls where that pod can be placed, but does not control where other pods are placed.
+
+Node affinity for stack pods is specified in the deployment's `spec.yml` file as follows:
+```
+node-affinities:
+  - label: nodetype
+    value: typeb
+```
+This example denotes that the stack's pods should only be placed on nodes that have the label `nodetype` with value `typeb`.
+### Node Taint Toleration
+K8s nodes can be given one or more "taints". These are special fields (distinct from labels) with a name (key) and optional value.
+When placing pods, the k8s scheduler will only assign a pod to a tainted node if the pod posesses a corresponding "toleration".
+This is metadata associated with the pod that specifies that the pod "tolerates" a given taint.
+Therefore taint toleration provides a mechanism by which only certain pods can be placed on specific nodes, and provides a complementary mechanism to node affinity.
+
+Taint toleration for stack pods is specified in the deployment's `spec.yml` file as follows:
+```
+node-tolerations:
+  - key: nodetype
+    value: typeb
+```
+This example denotes that the stack's pods will tolerate a taint: `nodetype=typeb`
+

docs/laconicd-with-console.md

@@ -1,9 +1,8 @@
 # Running a laconicd fixturenet with console
 
-The following tutorial explains the steps to run a laconicd fixturenet with CLI and web console that displays records in the registry. It is designed as an introduction to Stack Orchestrator and to showcase one component of the Laconic Stack. Prior to Stack Orchestrator, the following 4 repositories had to be cloned and setup manually:
+The following tutorial explains the steps to run a laconicd fixturenet with CLI and web console that displays records in the registry. It is designed as an introduction to Stack Orchestrator and to showcase one component of the Laconic Stack. Prior to Stack Orchestrator, the following repositories had to be cloned and setup manually:
 
 - https://git.vdb.to/cerc-io/laconicd
-- https://git.vdb.to/cerc-io/laconic-sdk
 - https://git.vdb.to/cerc-io/laconic-registry-cli
 - https://git.vdb.to/cerc-io/laconic-console
 
@@ -11,7 +10,7 @@ Now, with Stack Orchestrator, it is a few quick commands. Additionally, the `doc
 
 ## Setup laconic-so
 
-To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the user experience, this tutorial is focused on using a fresh Digital Ocean (DO) droplet with similar specs: 
+To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the user experience, this tutorial is focused on using a fresh Digital Ocean (DO) droplet with similar specs:
 16 GB Memory / 8 Intel vCPUs / 160 GB Disk.
 
 1. Login to the droplet as root (either by SSH key or password set in the DO console)
@@ -51,7 +50,7 @@ To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the
 1. Get the repositories
 
     ```
-    laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
+    laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd
     ```
 
 1. Build the containers:
@@ -76,6 +75,8 @@ To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the
 1. Create a deployment directory for the stack:
     ```
     laconic-so --stack fixturenet-laconic-loaded deploy init --output laconic-loaded.spec --map-ports-to-host any-same --config LACONIC_HOSTED_ENDPOINT=$BACKEND_ENDPOINT
+
+    # Update port mapping in the laconic-loaded.spec file to resolve port conflicts on host if any
     ```
     ```
     laconic-so --stack fixturenet-laconic-loaded deploy create --deployment-dir laconic-loaded-deployment --spec-file laconic-loaded.spec
@@ -95,52 +96,51 @@ To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the
     You'll see output from `laconicd` and the block height should be >1 to confirm it is running:
 
     ```
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:29PM INF indexed block exents height=12 module=txindex server=node
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF Timed out dur=4976.960115 height=13 module=consensus round=0 server=node step=1
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4","parts":{"hash":"22411A20B7F14CDA33244420FBDDAF24450C0628C7A06034FF22DAC3699DDCC8","total":1}},"height":13,"pol_round":-1,"round":0,"signature":"DEuqnaQmvyYbUwckttJmgKdpRu6eVm9i+9rQ1pIrV2PidkMNdWRZBLdmNghkIrUzGbW8Xd7UVJxtLRmwRASgBg==","timestamp":"2023-04-18T21:30:01.49450663Z"} server=node
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received complete proposal block hash=D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4 height=13 module=consensus server=node
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF finalizing commit of block hash={} height=13 module=consensus num_txs=0 root=1A8CA1AF139CCC80EC007C6321D8A63A46A793386EE2EDF9A5CA0AB2C90728B7 server=node
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF minted coins from module account amount=2059730459416582643aphoton from=mint module=x/bank
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF executed block height=13 module=state num_invalid_txs=0 num_valid_txs=0 server=node
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF commit synced commit=436F6D6D697449447B5B363520313037203630203232372039352038352032303820313334203231392032303520313433203130372031343920313431203139203139322038362031323720362031383520323533203137362031333820313735203135392031383620323334203135382031323120313431203230342037335D3A447D
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF committed state app_hash=416B3CE35F55D086DBCD8F6B958D13C0567F06B9FDB08AAF9FBAEA9E798DCC49 height=13 module=state num_txs=0 server=node
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF indexed block exents height=13 module=txindex server=node
+    laconicd-1         | 6:12AM INF indexed block events height=16 module=txindex
+    laconicd-1         | 6:12AM INF Timed out dur=2993.893332 height=17 module=consensus round=0 step=RoundStepNewHeight
+    laconicd-1         | 6:12AM INF received proposal module=consensus proposal="Proposal{17/0 (E15D03C180CE607AE8340A1325A0C134DFB4E1ADD992E173C701EBD362523267:1:DF138772FEF0, -1) 6A6F3B0A42B3 @ 2024-07-25T06:12:31.952967053Z}" proposer=86970D950BC9C16F3991A52D9C6DC55BA478A7C6
+    laconicd-1         | 6:12AM INF received complete proposal block hash=E15D03C180CE607AE8340A1325A0C134DFB4E1ADD992E173C701EBD362523267 height=17 module=consensus
+    laconicd-1         | 6:12AM INF finalizing commit of block hash=E15D03C180CE607AE8340A1325A0C134DFB4E1ADD992E173C701EBD362523267 height=17 module=consensus num_txs=0 root=AF4941107DC718ED1425E77A3DC7F1154FB780B7A7DE20288DC43442203527E3
+    laconicd-1         | 6:12AM INF finalized block block_app_hash=26A665360BB1EE64E54F97F2A5AB7F621B33A86D9896574000C05DE63F43F788 height=17 module=state num_txs_res=0 num_val_updates=0
+    laconicd-1         | 6:12AM INF executed block app_hash=26A665360BB1EE64E54F97F2A5AB7F621B33A86D9896574000C05DE63F43F788 height=17 module=state
+    laconicd-1         | 6:12AM INF committed state block_app_hash=AF4941107DC718ED1425E77A3DC7F1154FB780B7A7DE20288DC43442203527E3 height=17 module=state
+    laconicd-1         | 6:12AM INF indexed block events height=17 module=txindex
     ```
 
 4. Confirm operation of the registry CLI:
 
    ```
-   laconic-so deployment --dir laconic-loaded-deployment exec cli "laconic cns status"
+   laconic-so deployment --dir laconic-loaded-deployment exec cli "laconic registry status"
    ```
 
    ```
    {
      "version": "0.3.0",
      "node": {
-       "id": "4216af2ac9f68bda33a38803fc1b5c9559312c1d",
+       "id": "6e072894aa1f5d9535a1127a0d7a7f8e65100a2c",
        "network": "laconic_9000-1",
        "moniker": "localtestnet"
      },
      "sync": {
-       "latest_block_hash": "1BDF4CB9AE2390DA65BCF997C83133C18014FCDDCAE03708488F0B56FCEEA429",
-       "latest_block_height": "5",
-       "latest_block_time": "2023-08-09 16:00:30.386903172 +0000 UTC",
-       "catching_up": false
+       "latestBlockHash": "260102C283D0411CFBA0270F7DC182650FFCA737A2F6F652A985F6065696F590",
+       "latestBlockHeight": "49",
+       "latestBlockTime": "2024-07-25 06:14:05.626744215 +0000 UTC",
+       "catchingUp": false
      },
      "validator": {
-       "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
-       "voting_power": "1000000000000000"
+       "address": "86970D950BC9C16F3991A52D9C6DC55BA478A7C6",
+       "votingPower": "1000000000000000"
      },
      "validators": [
        {
-         "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
-         "voting_power": "1000000000000000",
-         "proposer_priority": "0"
+         "address": "86970D950BC9C16F3991A52D9C6DC55BA478A7C6",
+         "votingPower": "1000000000000000",
+         "proposerPriority": "0"
        }
      ],
-     "num_peers": "0",
+     "numPeers": "0",
      "peers": [],
-     "disk_usage": "292.0K"
+     "diskUsage": "688K"
    }
    ```
 
@@ -186,13 +186,13 @@ wns
 1. The following command will create a bond and publish a record:
 
 ```
-laconic-so --stack fixturenet-laconic-loaded deploy exec cli ./scripts/create-demo-records.sh
+laconic-so deployment --dir laconic-loaded-deployment exec cli ./scripts/create-demo-records.sh
 ```
 
 You'll get an output like:
 
 ```
-Balance is: 99998999999999998999600000
+Balance is: 9.9999e+25
 Created bond with id: dd88e8d6f9567b32b28e70552aea4419c5dd3307ebae85a284d1fe38904e301a
 Published demo-record-1.yml with id: bafyreierh3xnfivexlscdwubvczmddsnf46uytyfvrbdhkjzztvsz6ruly
 ```
@@ -223,5 +223,5 @@ record:
 - e.g,:
 
 ```
-laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns record list"
+laconic-so deployment --dir laconic-loaded-deployment exec cli "laconic registry record list"
 ```

requirements.txt

@@ -11,3 +11,5 @@ tomli==2.0.1
 validators==0.22.0
 kubernetes>=28.1.0
 humanfriendly>=10.0
+python-gnupg>=0.5.2
+requests>=2.3.2

setup.py

@@ -4,9 +4,11 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 with open("requirements.txt", "r", encoding="utf-8") as fh:
     requirements = fh.read()
+with open("stack_orchestrator/data/version.txt", "r", encoding="utf-8") as fh:
+    version = fh.readlines()[-1].strip(" \n")
 setup(
     name='laconic-stack-orchestrator',
-    version='1.0.12',
+    version=version,
     author='Cerc',
     author_email='info@cerc.io',
     license='GNU Affero General Public License',

stack_orchestrator/build/build_util.py

@@ -21,11 +21,6 @@ from stack_orchestrator.util import get_parsed_stack_config, warn_exit
 
 def get_containers_in_scope(stack: str):
 
-    # See: https://stackoverflow.com/a/20885799/1701505
-    from stack_orchestrator import data
-    with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
-        all_containers = container_list_file.read().splitlines()
-
     containers_in_scope = []
     if stack:
         stack_config = get_parsed_stack_config(stack)
@@ -33,11 +28,14 @@ def get_containers_in_scope(stack: str):
             warn_exit(f"stack {stack} does not define any containers")
         containers_in_scope = stack_config['containers']
     else:
-        containers_in_scope = all_containers
+        # See: https://stackoverflow.com/a/20885799/1701505
+        from stack_orchestrator import data
+        with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
+            containers_in_scope = container_list_file.read().splitlines()
 
     if opts.o.verbose:
         print(f'Containers: {containers_in_scope}')
         if stack:
             print(f"Stack: {stack}")
 
-    return containers_in_scope
+    return containers_in_scope

stack_orchestrator/constants.py

@@ -34,5 +34,8 @@ volumes_key = "volumes"
 security_key = "security"
 annotations_key = "annotations"
 labels_key = "labels"
+replicas_key = "replicas"
+node_affinities_key = "node-affinities"
+node_tolerations_key = "node-tolerations"
 kind_config_filename = "kind-config.yml"
 kube_config_filename = "kubeconfig.yml"

stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml

@@ -2,10 +2,11 @@ services:
   laconicd:
     restart: unless-stopped
     image: cerc/laconicd:local
-    command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
+    command: ["bash", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
     environment:
-      TEST_AUCTION_ENABLED: ${TEST_AUCTION_ENABLED}
-      TEST_REGISTRY_EXPIRY: ${TEST_REGISTRY_EXPIRY}
+      TEST_AUCTION_ENABLED: ${TEST_AUCTION_ENABLED:-false}
+      TEST_REGISTRY_EXPIRY: ${TEST_REGISTRY_EXPIRY:-false}
+      ONBOARDING_ENABLED: ${ONBOARDING_ENABLED:-false}
     volumes:
       # The cosmos-sdk node's database directory:
       - laconicd-data:/root/.laconicd
@@ -19,11 +20,9 @@ services:
       - "26657"
       - "26656"
       - "9473"
-      - "8545"
-      - "8546"
       - "9090"
-      - "9091"
       - "1317"
+
   cli:
     image: cerc/laconic-registry-cli:local
     volumes:

stack_orchestrator/data/compose/docker-compose-laconic-explorer.yml

@@ -0,0 +1,10 @@
+services:
+  laconic-explorer:
+    restart: unless-stopped
+    image: cerc/ping-pub:local
+    environment:
+      - LACONIC_LACONICD_API_URL=${LACONIC_LACONICD_API_URL:-http://localhost:1317}
+      - LACONIC_LACONICD_RPC_URL=${LACONIC_LACONICD_RPC_URL:-http://localhost:26657}
+      - LACONIC_LACONICD_CHAIN_ID=${LACONIC_LACONICD_CHAIN_ID:-chain-id-not-set}
+    ports:
+      - "5173"

stack_orchestrator/data/compose/docker-compose-snowballtools-base-backend.yml

@@ -1,13 +0,0 @@
-services:
-  snowballtools-base-backend:
-    image: cerc/snowballtools-base-backend:local
-    restart: always
-    volumes:
-      - data:/data
-      - config:/config:ro
-    ports:
-      - 8000
-
-volumes:
-  data:
-  config:

stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml

@@ -60,6 +60,7 @@ services:
     volumes:
       - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
       - ../config/watcher-ajna/start-server.sh:/app/start-server.sh
+      - ajna_watcher_gql_logs_data:/app/gql-logs
     ports:
       - "3008"
       - "9001"
@@ -74,3 +75,4 @@ services:
 
 volumes:
   ajna_watcher_db_data:
+  ajna_watcher_gql_logs_data:

stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml

@@ -74,8 +74,10 @@ services:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/azimuth-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/azimuth-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/azimuth-watcher/start-server.sh
+      - azimuth_watcher_gql_logs_data:/app/packages/azimuth-watcher/gql-logs
     ports:
       - "3001"
+      - "9001"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3001"]
       interval: 20s
@@ -135,8 +137,10 @@ services:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/censures-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/censures-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/censures-watcher/start-server.sh
+      - censures_watcher_gql_logs_data:/app/packages/censures-watcher/gql-logs
     ports:
       - "3002"
+      - "9003"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3002"]
       interval: 20s
@@ -196,8 +200,10 @@ services:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/claims-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/claims-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/claims-watcher/start-server.sh
+      - claims_watcher_gql_logs_data:/app/packages/claims-watcher/gql-logs
     ports:
       - "3003"
+      - "9005"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3003"]
       interval: 20s
@@ -257,8 +263,10 @@ services:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/conditional-star-release-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/conditional-star-release-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/conditional-star-release-watcher/start-server.sh
+      - conditional_star_release_watcher_gql_logs_data:/app/packages/conditional-star-release-watcher/gql-logs
     ports:
       - "3004"
+      - "9007"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3004"]
       interval: 20s
@@ -318,8 +326,10 @@ services:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/delegated-sending-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/delegated-sending-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/delegated-sending-watcher/start-server.sh
+      - delegated_sending_watcher_gql_logs_data:/app/packages/delegated-sending-watcher/gql-logs
     ports:
       - "3005"
+      - "9009"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3005"]
       interval: 20s
@@ -379,8 +389,10 @@ services:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/ecliptic-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/ecliptic-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/ecliptic-watcher/start-server.sh
+      - ecliptic_watcher_gql_logs_data:/app/packages/ecliptic-watcher/gql-logs
     ports:
       - "3006"
+      - "9011"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3006"]
       interval: 20s
@@ -440,8 +452,10 @@ services:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/linear-star-release-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/linear-star-release-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/linear-star-release-watcher/start-server.sh
+      - linear_star_release_watcher_gql_logs_data:/app/packages/linear-star-release-watcher/gql-logs
     ports:
       - "3007"
+      - "9013"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3007"]
       interval: 20s
@@ -501,8 +515,10 @@ services:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/polls-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/polls-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/polls-watcher/start-server.sh
+      - polls_watcher_gql_logs_data:/app/packages/polls-watcher/gql-logs
     ports:
       - "3008"
+      - "9015"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
       interval: 20s
@@ -552,3 +568,11 @@ services:
 
 volumes:
   watcher_db_data:
+  azimuth_watcher_gql_logs_data:
+  censures_watcher_gql_logs_data:
+  claims_watcher_gql_logs_data:
+  conditional_star_release_watcher_gql_logs_data:
+  delegated_sending_watcher_gql_logs_data:
+  ecliptic_watcher_gql_logs_data:
+  linear_star_release_watcher_gql_logs_data:
+  polls_watcher_gql_logs_data:

stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml

@@ -60,6 +60,7 @@ services:
     volumes:
       - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
       - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh
+      - merkl_sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
     ports:
       - "127.0.0.1:3007:3008"
       - "9003:9001"
@@ -74,3 +75,4 @@ services:
 
 volumes:
   merkl_sushiswap_v3_watcher_db_data:
+  merkl_sushiswap_v3_watcher_gql_logs_data:

stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml

@@ -60,6 +60,7 @@ services:
     volumes:
       - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
       - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh
+      - sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
     ports:
       - "127.0.0.1:3008:3008"
       - "9001:9001"
@@ -74,3 +75,4 @@ services:
 
 volumes:
   sushiswap_v3_watcher_db_data:
+  sushiswap_v3_watcher_gql_logs_data:

stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh

@@ -8,68 +8,68 @@ KEY="mykey"
 CHAINID="laconic_9000-1"
 MONIKER="localtestnet"
 KEYRING="test"
-KEYALGO="eth_secp256k1"
-LOGLEVEL="info"
-# trace evm
-TRACE="--trace"
-# TRACE=""
+KEYALGO="secp256k1"
+LOGLEVEL="${LOGLEVEL:-info}"
+DENOM="alnt"
+
 
 if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then
   # validate dependencies are installed
-  command -v jq > /dev/null 2>&1 || { echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"; exit 1; }
+  command -v jq > /dev/null 2>&1 || {
+    echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"
+    exit 1
+  }
 
   # remove existing daemon and client
   rm -rf $HOME/.laconicd/*
-  rm -rf $HOME/.laconic/*
 
   if [ -n "`which make`" ]; then
     make install
   fi
 
-  laconicd config keyring-backend $KEYRING
-  laconicd config chain-id $CHAINID
+  laconicd config set client chain-id $CHAINID
+  laconicd config set client keyring-backend $KEYRING
 
   # if $KEY exists it should be deleted
   laconicd keys add $KEY --keyring-backend $KEYRING --algo $KEYALGO
 
   # Set moniker and chain-id for Ethermint (Moniker can be anything, chain-id must be an integer)
-  laconicd init $MONIKER --chain-id $CHAINID
+  laconicd init $MONIKER --chain-id $CHAINID --default-denom $DENOM
 
-  # Change parameter token denominations to aphoton
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["staking"]["params"]["bond_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["crisis"]["constant_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["gov"]["deposit_params"]["min_deposit"][0]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["mint"]["params"]["mint_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  # Custom modules
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commit_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveal_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_minimum_bid"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  update_genesis() {
+    jq "$1" $HOME/.laconicd/config/genesis.json > $HOME/.laconicd/config/tmp_genesis.json &&
+      mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  }
 
   if [[ "$TEST_REGISTRY_EXPIRY" == "true" ]]; then
     echo "Setting timers for expiry tests."
 
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    update_genesis '.app_state["registry"]["params"]["record_rent_duration"]="60s"'
+    update_genesis '.app_state["registry"]["params"]["authority_grace_period"]="60s"'
+    update_genesis '.app_state["registry"]["params"]["authority_rent_duration"]="60s"'
   fi
 
   if [[ "$TEST_AUCTION_ENABLED" == "true" ]]; then
     echo "Enabling auction and setting timers."
 
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_enabled"]=true' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="300s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commits_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveals_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    update_genesis '.app_state["registry"]["params"]["authority_auction_enabled"]=true'
+    update_genesis '.app_state["registry"]["params"]["authority_rent_duration"]="60s"'
+    update_genesis '.app_state["registry"]["params"]["authority_grace_period"]="300s"'
+    update_genesis '.app_state["registry"]["params"]["authority_auction_commits_duration"]="60s"'
+    update_genesis '.app_state["registry"]["params"]["authority_auction_reveals_duration"]="60s"'
+  fi
+
+  if [[ "$ONBOARDING_ENABLED" == "true" ]]; then
+    echo "Enabling validator onboarding."
+
+    update_genesis '.app_state["onboarding"]["params"]["onboarding_enabled"]=true'
   fi
 
   # increase block time (?)
-  cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["time_iota_ms"]="1000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  update_genesis '.consensus["params"]["block"]["time_iota_ms"]="1000"'
 
   # Set gas limit in genesis
-  cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["max_gas"]="10000000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  update_genesis '.consensus["params"]["block"]["max_gas"]="10000000"'
 
   # disable produce empty block
   if [[ "$OSTYPE" == "darwin"* ]]; then
@@ -78,30 +78,6 @@ if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then
       sed -i 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml
   fi
 
-  if [[ $1 == "pending" ]]; then
-    if [[ "$OSTYPE" == "darwin"* ]]; then
-        sed -i '' 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml
-    else
-        sed -i 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml
-    fi
-  fi
-
   # Enable telemetry (prometheus metrics: http://localhost:1317/metrics?format=prometheus)
   if [[ "$OSTYPE" == "darwin"* ]]; then
     sed -i '' 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml
@@ -114,23 +90,27 @@ if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then
   fi
 
   # Allocate genesis accounts (cosmos formatted addresses)
-  laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING
+  # 10^30 alnt | 10^12 lnt
+  laconicd genesis add-genesis-account $KEY 1000000000000000000000000000000$DENOM --keyring-backend $KEYRING
 
   # Sign genesis transaction
-  laconicd gentx $KEY 1000000000000000000000aphoton --keyring-backend $KEYRING --chain-id $CHAINID
+  # 10^24 alnt | 10^6 lnt
+  laconicd genesis gentx $KEY 1000000000000000000000000$DENOM --keyring-backend $KEYRING --chain-id $CHAINID
 
   # Collect genesis tx
-  laconicd collect-gentxs
+  laconicd genesis collect-gentxs
 
   # Run this to ensure everything worked and that the genesis file is setup correctly
-  laconicd validate-genesis
-
-  if [[ $1 == "pending" ]]; then
-    echo "pending mode is on, please wait for the first block committed."
-  fi
+  laconicd genesis validate
 else
   echo "Using existing database at $HOME/.laconicd.  To replace, run '`basename $0` clean'"
 fi
 
 # Start the node (remove the --pruning=nothing flag if historical queries are not needed)
-laconicd start --pruning=nothing --evm.tracer=json $TRACE --log_level $LOGLEVEL --minimum-gas-prices=0.0001aphoton --json-rpc.api eth,txpool,personal,net,debug,web3,miner --api.enable --gql-server --gql-playground
+laconicd start \
+  --pruning=nothing \
+  --log_level $LOGLEVEL \
+  --minimum-gas-prices=1$DENOM \
+  --api.enable \
+  --rpc.laddr="tcp://0.0.0.0:26657" \
+  --gql-server --gql-playground

stack_orchestrator/data/config/fixturenet-laconicd/registry-cli-config-template.yml

@@ -1,9 +1,9 @@
 services:
-  cns:
-    restEndpoint: 'http://laconicd:1317'
+  registry:
+    rpcEndpoint: 'http://laconicd:26657'
     gqlEndpoint: 'http://laconicd:9473/api'
     userKey: REPLACE_WITH_MYKEY
     bondId:
     chainId: laconic_9000-1
     gas: 350000
-    fees: 200000aphoton
+    fees: 2000000alnt

stack_orchestrator/data/config/mainnet-laconicd/registry-cli-config-template.yml

@@ -1,9 +1,9 @@
 services:
-  cns:
-    restEndpoint: 'http://laconicd:1317'
+  registry:
+    rpcEndpoint: 'http://laconicd:26657'
     gqlEndpoint: 'http://laconicd:9473/api'
     userKey: REPLACE_WITH_MYKEY
     bondId:
     chainId: laconic_9000-1
     gas: 250000
-    fees: 200000aphoton
+    fees: 2000000alnt

stack_orchestrator/data/config/mainnet-laconicd/scripts/run-laconicd.sh

@@ -1,18 +1,15 @@
 #!/bin/sh
-if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
 #TODO: pass these in from the caller
-TRACE="--trace"
 LOGLEVEL="info"
 
 laconicd start \
     --pruning=nothing \
-    --evm.tracer=json $TRACE \
     --log_level $LOGLEVEL \
-    --minimum-gas-prices=0.0001aphoton \
-    --json-rpc.api eth,txpool,personal,net,debug,web3,miner \
+    --minimum-gas-prices=1alnt \
     --api.enable \
     --gql-server \
     --gql-playground

stack_orchestrator/data/config/monitoring/grafana/dashboards/subgraphs-dashboard.json

@@ -18,7 +18,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 6,
+  "id": 39,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -1202,6 +1202,640 @@
       ],
       "title": "ETH RPC failed requests rate",
       "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 24,
+        "x": 0,
+        "y": 22
+      },
+      "id": 32,
+      "options": {
+        "code": {
+          "language": "plaintext",
+          "showLineNumbers": false,
+          "showMiniMap": false
+        },
+        "content": "<center style=\"margin-top: 1rem\"> \n  <h1>GraphQL Server</h1> \n</center>",
+        "mode": "html"
+      },
+      "pluginVersion": "10.2.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "refId": "A"
+        }
+      ],
+      "transparent": true,
+      "type": "text"
+    },
+    {
+      "cards": {},
+      "color": {
+        "cardColor": "#b4ff00",
+        "colorScale": "sqrt",
+        "colorScheme": "interpolatePlasma",
+        "exponent": 0.5,
+        "mode": "spectrum"
+      },
+      "dataFormat": "tsbuckets",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "scaleDistribution": {
+              "type": "linear"
+            }
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 0,
+        "y": 24
+      },
+      "heatmap": {},
+      "hideZeroBuckets": true,
+      "highlightCards": true,
+      "id": 29,
+      "legend": {
+        "show": false
+      },
+      "links": [],
+      "options": {
+        "calculate": false,
+        "calculation": {},
+        "cellGap": 2,
+        "cellValues": {},
+        "color": {
+          "exponent": 0.5,
+          "fill": "#b4ff00",
+          "mode": "scheme",
+          "reverse": false,
+          "scale": "exponential",
+          "scheme": "Plasma",
+          "steps": 128
+        },
+        "exemplars": {
+          "color": "rgba(255,0,255,0.7)"
+        },
+        "filterValues": {
+          "le": 1e-9
+        },
+        "legend": {
+          "show": false
+        },
+        "rowsFrame": {
+          "layout": "auto"
+        },
+        "showValue": "never",
+        "tooltip": {
+          "show": true,
+          "showColorScale": false,
+          "yHistogram": false
+        },
+        "yAxis": {
+          "axisPlacement": "left",
+          "reverse": false,
+          "unit": "short"
+        }
+      },
+      "pluginVersion": "10.2.3",
+      "repeatDirection": "v",
+      "reverseYBuckets": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(query_execution_time_bucket{deployment=\"[[subgraph_hash]]\"}[1m])) by (le)",
+          "format": "heatmap",
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{le}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Query Execution Time Histogram ([[subgraph_name]])",
+      "tooltip": {
+        "show": true,
+        "showHistogram": false
+      },
+      "type": "heatmap",
+      "xAxis": {
+        "show": true
+      },
+      "yAxis": {
+        "format": "short",
+        "logBase": 1,
+        "show": true
+      },
+      "yBucketBound": "auto"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "fillOpacity": 80,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineWidth": 1
+          },
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 6,
+        "y": 24
+      },
+      "id": 26,
+      "links": [],
+      "options": {
+        "bucketOffset": 0,
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        }
+      },
+      "pluginVersion": "10.2.3",
+      "repeat": "subgraph_hash",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(query_execution_time_bucket{deployment=\"[[subgraph_hash]]\"}[1m])) by (le)",
+          "format": "heatmap",
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{le}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Query Execution Time Histogram ([[subgraph_name]])",
+      "type": "histogram"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 80,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "id": 36,
+      "links": [],
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.2.3",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "sum(increase(query_execution_time_count{deployment=\"$subgraph_hash\"}[1m]))",
+          "format": "time_series",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{subgraph_deployment}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Queries  ([[subgraph_name]])",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "Queries per minute per minute",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "id": 27,
+      "links": [],
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.2.3",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(query_execution_time_count[1m])) by (deployment)",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{deployment}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Query Rate (All Subgraphs)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 80,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "id": 67,
+      "links": [],
+      "options": {
+        "legend": {
+          "calcs": [
+            "sum"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.2.3",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(query_execution_time_count[1m])) by (deployment)",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{deployment}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Queries  (All Subgraphs)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 70,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "locale"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 40
+      },
+      "id": 41,
+      "links": [],
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.2.3",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "increase(query_execution_time_count{status=\"failed\"}[1m])",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{deployment}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Failed Queries  (All Subgraphs)",
+      "type": "timeseries"
     }
   ],
   "refresh": "10s",

stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json

@@ -18,7 +18,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 6,
+  "id": 2,
   "links": [
     {
       "asDropdown": false,
@@ -117,7 +117,7 @@
               "disableTextWrap": false,
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(watcher_info{job=\"$job\", instance=\"$watcher\"}) by (version, commitHash)",
+              "expr": "max(watcher_info{job=\"$job\", instance=\"$watcher\"}) by (repository, version, commitHash)",
               "format": "table",
               "fullMetaSearch": false,
               "includeNullMetadata": true,
@@ -139,12 +139,14 @@
                 "includeByName": {},
                 "indexByName": {
                   "Time": 0,
-                  "Value": 3,
-                  "commitHash": 2,
-                  "version": 1
+                  "Value": 4,
+                  "commitHash": 3,
+                  "repository": 1,
+                  "version": 2
                 },
                 "renameByName": {
                   "commitHash": "Commit hash",
+                  "repository": "Repository",
                   "version": "Release"
                 }
               }
@@ -280,7 +282,7 @@
           },
           "gridPos": {
             "h": 3,
-            "w": 4,
+            "w": 3,
             "x": 0,
             "y": 4
           },
@@ -301,6 +303,75 @@
             "wideLayout": true
           },
           "pluginVersion": "10.2.3",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "disableTextWrap": false,
+              "editorMode": "code",
+              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_processed\"}",
+              "fullMetaSearch": false,
+              "hide": false,
+              "includeNullMetadata": true,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "B",
+              "useBackend": false
+            }
+          ],
+          "title": "Event processed block",
+          "type": "stat"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "fixedColor": "blue",
+                "mode": "shades"
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 3,
+            "w": 3,
+            "x": 3,
+            "y": 4
+          },
+          "id": 38,
+          "options": {
+            "colorMode": "background",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "textMode": "auto",
+            "wideLayout": true
+          },
+          "pluginVersion": "10.2.3",
           "targets": [
             {
               "datasource": {
@@ -320,7 +391,7 @@
               "useBackend": false
             }
           ],
-          "title": "Latest indexed block",
+          "title": "Inserted processed block",
           "type": "stat"
         },
         {
@@ -350,8 +421,8 @@
           },
           "gridPos": {
             "h": 3,
-            "w": 4,
-            "x": 4,
+            "w": 3,
+            "x": 6,
             "y": 4
           },
           "id": 11,
@@ -419,8 +490,8 @@
           },
           "gridPos": {
             "h": 3,
-            "w": 4,
-            "x": 8,
+            "w": 3,
+            "x": 9,
             "y": 4
           },
           "id": 12,
@@ -637,7 +708,7 @@
                 "uid": "PBFA97CFB590B2093"
               },
               "editorMode": "code",
-              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}",
+              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_processed\"}",
               "hide": false,
               "instant": false,
               "legendFormat": "{{__name__}}",
@@ -748,7 +819,7 @@
                 "uid": "PBFA97CFB590B2093"
               },
               "editorMode": "code",
-              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}",
+              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_processed\"}",
               "hide": false,
               "instant": false,
               "legendFormat": "{{__name__}}",
@@ -934,12 +1005,12 @@
                 "uid": "PBFA97CFB590B2093"
               },
               "disableTextWrap": false,
-              "editorMode": "builder",
-              "expr": "last_processed_block_number{job=~\"$job\", instance=~\"$watcher\"}",
+              "editorMode": "code",
+              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_processed\"}",
               "fullMetaSearch": false,
               "includeNullMetadata": true,
               "instant": false,
-              "legendFormat": "latest_processed",
+              "legendFormat": "{{kind}}",
               "range": true,
               "refId": "A",
               "useBackend": false
@@ -950,7 +1021,7 @@
                 "uid": "PBFA97CFB590B2093"
               },
               "disableTextWrap": false,
-              "editorMode": "builder",
+              "editorMode": "code",
               "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_canonical\"}",
               "fullMetaSearch": false,
               "hide": false,
@@ -1393,7 +1464,7 @@
                 "uid": "PBFA97CFB590B2093"
               },
               "editorMode": "code",
-              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}",
+              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_processed\"}",
               "hide": false,
               "instant": false,
               "legendFormat": "{{__name__}}",
@@ -2099,6 +2170,282 @@
           ],
           "type": "table"
         },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 6,
+            "w": 8,
+            "x": 0,
+            "y": 33
+          },
+          "id": 35,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "gql_query_count_total{job=~\"$job\", instance=~\"$watcher\"}",
+              "instant": false,
+              "legendFormat": "{{__name__}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Total GQL query count",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 6,
+            "w": 8,
+            "x": 8,
+            "y": 33
+          },
+          "id": 36,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "gql_query_count{job=~\"$job\", instance=~\"$watcher\"}",
+              "instant": false,
+              "legendFormat": "{{name}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "GQL query count",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 6,
+            "w": 8,
+            "x": 16,
+            "y": 33
+          },
+          "id": 37,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "gql_query_duration_seconds{job=~\"$job\", instance=~\"$watcher\"}",
+              "instant": false,
+              "legendFormat": "{{name}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "GQL queries duration",
+          "type": "timeseries"
+        },
         {
           "datasource": {
             "type": "prometheus",
@@ -2138,7 +2485,7 @@
             "h": 5,
             "w": 8,
             "x": 0,
-            "y": 33
+            "y": 39
           },
           "id": 18,
           "options": {
@@ -2284,7 +2631,7 @@
             "h": 5,
             "w": 8,
             "x": 8,
-            "y": 33
+            "y": 39
           },
           "id": 17,
           "options": {
@@ -2432,7 +2779,7 @@
             "h": 5,
             "w": 8,
             "x": 16,
-            "y": 33
+            "y": 39
           },
           "id": 19,
           "options": {
@@ -2599,7 +2946,7 @@
             "h": 6,
             "w": 8,
             "x": 0,
-            "y": 38
+            "y": 44
           },
           "hide": true,
           "id": 8,
@@ -2697,7 +3044,7 @@
             "h": 6,
             "w": 8,
             "x": 8,
-            "y": 38
+            "y": 44
           },
           "id": 9,
           "options": {
@@ -2794,7 +3141,7 @@
             "h": 6,
             "w": 8,
             "x": 16,
-            "y": 38
+            "y": 44
           },
           "id": 10,
           "options": {
@@ -2937,6 +3284,6 @@
   "timepicker": {},
   "timezone": "",
   "title": "Watchers",
-  "version": 3,
+  "version": 5,
   "weekStart": ""
 }

stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml

@@ -849,3 +849,81 @@ groups:
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
+      
+      # Secured Finance
+      - uid: secured_finance_diff_external
+        title: secured_finance_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="secured-finance", instance="secured-finance", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false

stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml

@@ -2,7 +2,6 @@
   host = "0.0.0.0"
   port = 3008
   kind = "active"
-  gqlPath = "/"
 
   # Checkpointing state.
   checkpointing = true
@@ -22,23 +21,30 @@
   # Interval in number of blocks at which to clear entities cache.
   clearEntitiesCacheInterval = 1000
 
-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
   # Flag to specify whether RPC endpoint supports block hash as block tag parameter
   rpcSupportsBlockHashParam = false
 
-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"
 
-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000
 
-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day
 
 [metrics]
   host = "0.0.0.0"
@@ -85,6 +91,9 @@
   # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
   blockDelayInMilliSecs = 30000
 
+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
   # Boolean to switch between modes of processing events when starting the server.
   # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
   # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).

stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml

@@ -1,6 +1,7 @@
 [server]
   host = "0.0.0.0"
-  maxSimultaneousRequests = -1
+  [server.gql]
+    maxSimultaneousRequests = -1
 
 [metrics]
   host = "0.0.0.0"

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml

@@ -2,7 +2,6 @@
   host = "0.0.0.0"
   port = 3008
   kind = "active"
-  gqlPath = '/'
 
   # Checkpointing state.
   checkpointing = true
@@ -22,23 +21,30 @@
   # Interval in number of blocks at which to clear entities cache.
   clearEntitiesCacheInterval = 1000
 
-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
   # Flag to specify whether RPC endpoint supports block hash as block tag parameter
   rpcSupportsBlockHashParam = false
 
-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"
 
-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000
 
-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day
 
 [metrics]
   host = "0.0.0.0"
@@ -85,6 +91,9 @@
   # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
   blockDelayInMilliSecs = 30000
 
+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
   # Boolean to switch between modes of processing events when starting the server.
   # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
   # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).

stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml

@@ -2,7 +2,6 @@
   host = "0.0.0.0"
   port = 3008
   kind = "active"
-  gqlPath = "/"
 
   # Checkpointing state.
   checkpointing = true
@@ -22,23 +21,30 @@
   # Interval in number of blocks at which to clear entities cache.
   clearEntitiesCacheInterval = 1000
 
-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
   # Flag to specify whether RPC endpoint supports block hash as block tag parameter
   rpcSupportsBlockHashParam = false
 
-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"
 
-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000
 
-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day
 
 [metrics]
   host = "0.0.0.0"
@@ -85,6 +91,9 @@
   # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
   blockDelayInMilliSecs = 30000
 
+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
   # Boolean to switch between modes of processing events when starting the server.
   # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
   # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).

stack_orchestrator/data/container-build/cerc-builder-js/Dockerfile

@@ -68,5 +68,5 @@ ENV PATH="${PATH}:/scripts"
 COPY entrypoint.sh .
 ENTRYPOINT ["./entrypoint.sh"]
 # Placeholder CMD : generally this will be overridden at run time like :
-# docker run -it -v /home/builder/cerc/laconic-sdk:/workspace cerc/builder-js sh -c 'cd /workspace && yarn && yarn build'
+# docker run -it -v /home/builder/cerc/registry-sdk:/workspace cerc/builder-js sh -c 'cd /workspace && yarn && yarn build'
 CMD node --version

stack_orchestrator/data/container-build/cerc-laconic-console-host/build.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Build cerc/laconic-registry-cli
+# Build cerc/laconic-console-host
 
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 

stack_orchestrator/data/container-build/cerc-laconic-registry-cli/create-demo-records.sh

@@ -5,16 +5,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
-registry_command="laconic cns"
+registry_command="laconic registry"
 demo_records_dir="scripts/demo-records"
 
 # Check we have funds
 funds_response=$(${registry_command} account get --address $(cat my-address.txt))
-funds_balance=$(echo ${funds_response} | jq -r .[0].balance[0].quantity)
+funds_balance=$(echo ${funds_response} | jq -r ".[0].balance[0].quantity")
 echo "Balance is: ${funds_balance}"
 
 # Create a bond
-bond_create_result=$(${registry_command} bond create --type aphoton --quantity 1000000000)
+bond_create_result=$(${registry_command} bond create --type alnt --quantity 1000000000)
 bond_id=$(echo ${bond_create_result} | jq -r .bondId)
 echo "Created bond with id: ${bond_id}"
 

stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-9.yml

@@ -7,9 +7,9 @@ record:
     env:
       ENV_VAR_A: A
       ENV_VAR_B: B
-  crn:
-    - crn://foo.bar
-    - crn://bar.baz
+  lrn:
+    - lrn://foo.bar
+    - lrn://bar.baz
   meta:
     foo: bar
   tags:

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -28,6 +28,8 @@ RUN \
     && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
     # Install pnpm
     && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
+    # Install bun
+    && su ${USERNAME} -c "umask 0002 && npm install -g bun@1.1.x" \
     && npm cache clean --force > /dev/null 2>&1
 
 # [Optional] Uncomment this section to install additional OS packages.

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh

@@ -14,6 +14,8 @@ if [ -z "$CERC_BUILD_TOOL" ]; then
     CERC_BUILD_TOOL=pnpm
   elif [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
+  elif [ -f "bun.lockb" ]; then
+    CERC_BUILD_TOOL=bun
   else
     CERC_BUILD_TOOL=npm
   fi

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -5,14 +5,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 
 CERC_MIN_NEXTVER=13.4.2
+CERC_DEFAULT_WEBPACK_VER="5.93.0"
 
 CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}"
+CERC_WEBPACK_VERSION="${CERC_WEBPACK_VERSION:-keep}"
+
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
   if [ -f "pnpm-lock.yaml" ]; then
     CERC_BUILD_TOOL=pnpm
   elif [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
+  elif [ -f "bun.lockb" ]; then
+    CERC_BUILD_TOOL=bun
   else
     CERC_BUILD_TOOL=npm
   fi
@@ -21,15 +26,29 @@ fi
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 WORK_DIR="${1:-/app}"
 
+if [ -f "${WORK_DIR}/build-webapp.sh" ]; then
+  echo "Building webapp with ${WORK_DIR}/build-webapp.sh ..."
 cd "${WORK_DIR}" || exit 1
 
+  ./build-webapp.sh || exit 1
+  exit 0
+fi
+
+if [ -f "next.config.mjs" ]; then
+  NEXT_CONFIG_JS="next.config.mjs"
+  IMPORT_OR_REQUIRE="import"
+else
+  NEXT_CONFIG_JS="next.config.js"
+  IMPORT_OR_REQUIRE="require"
+fi
+
 # If this file doesn't exist at all, we'll get errors below.
-if [ ! -f "next.config.js" ]; then
-  touch next.config.js
+if [ ! -f "${NEXT_CONFIG_JS}" ]; then
+  touch ${NEXT_CONFIG_JS}
 fi
 
 if [ ! -f "next.config.dist" ]; then
-  cp next.config.js next.config.dist
+  cp $NEXT_CONFIG_JS next.config.dist
 fi
 
 which js-beautify >/dev/null
@@ -37,17 +56,34 @@ if [ $? -ne 0 ]; then
   npm i -g js-beautify
 fi
 
-js-beautify next.config.dist > next.config.js
-echo "" >> next.config.js
+# js-beautify formats NEXTJS_CONFIG_FILE (ie next.config.js / next.config.mjs) so we can reliably transformable later
+js-beautify next.config.dist > ${NEXT_CONFIG_JS}
+echo "" >> ${NEXT_CONFIG_JS}
 
-WEBPACK_REQ_LINE=$(grep -n "require([\'\"]webpack[\'\"])" next.config.js | cut -d':' -f1)
-if [ -z "$WEBPACK_REQ_LINE" ]; then
-  cat > next.config.js.0 <<EOF
+if [ "${IMPORT_OR_REQUIRE}" == "require" ]; then
+  WEBPACK_REQ_LINE=$(grep -n "require([\'\"]webpack[\'\"])" ${NEXT_CONFIG_JS} | cut -d':' -f1)
+  if [ -z "$WEBPACK_REQ_LINE" ]; then
+    cat > ${NEXT_CONFIG_JS}.0 <<EOF
     const webpack = require('webpack');
 EOF
+  fi
+else
+  WEBPACK_IMPORT_LINE=$(grep -n "^import .*[\'\"]webpack[\'\"];?$" ${NEXT_CONFIG_JS} | cut -d':' -f1)
+  if [ -z "$WEBPACK_IMPORT_LINE" ]; then
+    cat > ${NEXT_CONFIG_JS}.0 <<EOF
+    import webpack from 'webpack';
+EOF
+  fi
+  CREATE_REQUIRE_LINE=$(grep -n "require = createRequire" ${NEXT_CONFIG_JS} | cut -d':' -f1)
+  if [ -z "$CREATE_REQUIRE_LINE" ]; then
+    cat >> ${NEXT_CONFIG_JS}.0 <<EOF
+    import { createRequire } from "module";
+    const require = createRequire(import.meta.url);
+EOF
+  fi
 fi
 
-cat > next.config.js.1 <<EOF
+cat > ${NEXT_CONFIG_JS}.1 <<EOF
 let envMap;
 try {
   // .env-list.json provides us a list of identifiers which should be replaced at runtime.
@@ -55,7 +91,8 @@ try {
     a[v] = \`"CERC_RUNTIME_ENV_\${v.split(/\./).pop()}"\`;
     return a;
   }, {});
-} catch {
+} catch (e) {
+  console.error(e);
   // If .env-list.json cannot be loaded, we are probably running in dev mode, so use process.env instead.
   envMap = Object.keys(process.env).reduce((a, v) => {
     if (v.startsWith('CERC_')) {
@@ -64,40 +101,80 @@ try {
     return a;
   }, {});
 }
+console.log(envMap);
 EOF
 
-CONFIG_LINES=$(wc -l next.config.js | awk '{ print $1 }')
-ENV_LINE=$(grep -n 'env:' next.config.js | cut -d':' -f1)
-WEBPACK_CONF_LINE=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d':' -f1)
-NEXT_SECTION_ADJUSTMENT=0
+grep 'withPWA' ${NEXT_CONFIG_JS} >/dev/null && HAS_WITHPWA=true || HAS_WITHPWA=false
 
-if [ -n "$WEBPACK_CONF_LINE" ]; then
-  WEBPACK_CONF_VAR=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d',' -f1 | cut -d'(' -f2)
-  head -$(( ${WEBPACK_CONF_LINE} )) next.config.js > next.config.js.2
-  cat > next.config.js.3 <<EOF
-      $WEBPACK_CONF_VAR.plugins.push(new webpack.DefinePlugin(envMap));
-EOF
-  NEXT_SECTION_LINE=$((WEBPACK_CONF_LINE))
-elif [ -n "$ENV_LINE" ]; then
-  head -$(( ${ENV_LINE} - 1 )) next.config.js > next.config.js.2
-  cat > next.config.js.3 <<EOF
-    webpack: (config) => {
+if [ "$HAS_WITHPWA" == "true" ]; then
+  if [ "$IMPORT_OR_REQUIRE" == "import" ]; then
+    cat > ${NEXT_CONFIG_JS}.2 <<EOF
+const __xPWA__ = (p) => {
+  const realPWA = withPWA(p);
+  return (nextConfig) => {
+    const modConfig = {...nextConfig};
+
+    modConfig.webpack = (config) => {
       config.plugins.push(new webpack.DefinePlugin(envMap));
-      return config;
-    },
+      return nextConfig.webpack ? nextConfig.webpack(config) : config;
+    };
+
+    return realPWA(modConfig);
+  };
+};
 EOF
-  NEXT_SECTION_ADJUSTMENT=1
-  NEXT_SECTION_LINE=$ENV_LINE
+  else
+    cat > ${NEXT_CONFIG_JS}.3 <<EOF
+const __xPWA__ = (nextConfig) => {
+  const modConfig = {...nextConfig};
+
+  modConfig.webpack = (config) => {
+    config.plugins.push(new webpack.DefinePlugin(envMap));
+    return nextConfig.webpack ? nextConfig.webpack(config) : config;
+  };
+
+  return withPWA(modConfig);
+};
+EOF
+  fi
+
+  cat ${NEXT_CONFIG_JS} | js-beautify | sed 's/withPWA(/__xPWA__(/g' > ${NEXT_CONFIG_JS}.4
 else
-  echo "WARNING: Cannot find location to insert environment variable map in next.config.js" 1>&2
-  rm -f next.config.js.*
-  NEXT_SECTION_LINE=0
+    cat > ${NEXT_CONFIG_JS}.3 <<EOF
+  const __xCfg__ = (nextConfig) => {
+    const modConfig = {...nextConfig};
+
+    modConfig.webpack = (config) => {
+      config.plugins.push(new webpack.DefinePlugin(envMap));
+      return nextConfig.webpack ? nextConfig.webpack(config) : config;
+    };
+
+    return modConfig;
+  };
+EOF
+  if [ "$IMPORT_OR_REQUIRE" == "import" ]; then
+    cat ${NEXT_CONFIG_JS} | js-beautify | sed 's/export\s\+default\s\+/const __orig_cfg__ = /g' > ${NEXT_CONFIG_JS}.4
+    echo "export default __xCfg__(__orig_cfg__);" > ${NEXT_CONFIG_JS}.5
+  else
+    cat ${NEXT_CONFIG_JS} | js-beautify | sed 's/module.exports\s\+=\s\+/const __orig_cfg__ = /g' > ${NEXT_CONFIG_JS}.4
+    echo "module.exports = __xCfg__(__orig_cfg__);" > ${NEXT_CONFIG_JS}.5
+  fi
 fi
 
-tail -$(( ${CONFIG_LINES} - ${NEXT_SECTION_LINE} + ${NEXT_SECTION_ADJUSTMENT} )) next.config.js > next.config.js.5
 
-cat next.config.js.* | sed 's/^ *//g' | js-beautify | grep -v 'process\.\env\.' | js-beautify > next.config.js
-rm next.config.js.*
+rm -f ${NEXT_CONFIG_JS}
+for ((i=0; i <= 10; i++)); do
+  if [ -s "${NEXT_CONFIG_JS}.${i}" ]; then
+    if [ $i -le 2 ] ; then
+      cat ${NEXT_CONFIG_JS}.${i} >> ${NEXT_CONFIG_JS}
+    else
+      cat ${NEXT_CONFIG_JS}.${i} | sed 's/^ *//g' | js-beautify | grep -v 'process\.\env\.' | js-beautify >> ${NEXT_CONFIG_JS}
+    fi
+  fi
+done
+rm ${NEXT_CONFIG_JS}.*
+cat ${NEXT_CONFIG_JS} | js-beautify > ${NEXT_CONFIG_JS}.pretty
+mv ${NEXT_CONFIG_JS}.pretty ${NEXT_CONFIG_JS}
 
 "${SCRIPT_DIR}/find-env.sh" "$(pwd)" > .env-list.json
 
@@ -105,8 +182,6 @@ if [ ! -f "package.dist" ]; then
   cp package.json package.dist
 fi
 
-cat package.dist | jq '.scripts.cerc_compile = "next experimental-compile"' | jq '.scripts.cerc_generate = "next experimental-generate"' > package.json
-
 CUR_NEXT_VERSION="`jq -r '.dependencies.next' package.json`"
 
 if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VERSION" ]; then
@@ -115,10 +190,38 @@ if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VE
   mv package.json.$$ package.json
 fi
 
+CUR_WEBPACK_VERSION="`jq -r '.dependencies.webpack' package.json`"
+if [ -z "$CUR_WEBPACK_VERSION" ]; then
+  CUR_WEBPACK_VERSION="`jq -r '.devDependencies.webpack' package.json`"
+fi
+if [ "${CERC_WEBPACK_VERSION}" != "keep" ] || [ "${CUR_WEBPACK_VERSION}" == "null" ]; then
+  if [ -z "$CERC_WEBPACK_VERSION" ] || [ "$CERC_WEBPACK_VERSION" == "keep" ]; then
+    CERC_WEBPACK_VERSION="${CERC_DEFAULT_WEBPACK_VER}"
+  fi
+  echo "Webpack is required for env variable substitution.  Adding to webpack@$CERC_WEBPACK_VERSION to dependencies..." 1>&2
+  cat package.json | jq ".dependencies.webpack = \"$CERC_WEBPACK_VERSION\"" > package.json.$$
+  mv package.json.$$ package.json
+fi
+
 time $CERC_BUILD_TOOL install || exit 1
 
 CUR_NEXT_VERSION=`jq -r '.version' node_modules/next/package.json`
 
+# See https://github.com/vercel/next.js/discussions/46544
+semver -p -r ">=14.2.0" "$CUR_NEXT_VERSION"
+if [ $? -eq 0 ]; then
+  # For >= 14.2.0
+  CERC_NEXT_COMPILE_COMMAND="next build --experimental-build-mode compile"
+  CERC_NEXT_GENERATE_COMMAND="next build --experimental-build-mode generate"
+else
+  # For 13.4.2 to 14.1.x
+  CERC_NEXT_COMPILE_COMMAND="next experimental-compile"
+  CERC_NEXT_GENERATE_COMMAND="next experimental-generate"
+fi
+
+cat package.json | jq ".scripts.cerc_compile = \"$CERC_NEXT_COMPILE_COMMAND\"" | jq ".scripts.cerc_generate = \"$CERC_NEXT_GENERATE_COMMAND\"" > package.json.$$
+mv package.json.$$ package.json
+
 semver -p -r ">=$CERC_MIN_NEXTVER" $CUR_NEXT_VERSION
 if [ $? -ne 0 ]; then
   cat <<EOF

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh

@@ -20,9 +20,11 @@ for d in $(find . -maxdepth 1 -type d | grep -v '\./\.' | grep '/' | cut -d'/' -
   done
 done
 
-NEXT_CONF="next.config.js next.config.dist"
+NEXT_CONF="next.config.mjs next.config.js next.config.dist"
 for f in $NEXT_CONF; do
-    cat "$f" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o 'process.env.[A-Za-z0-9_]+' >> $TMPF
+    if [ -f "$f" ]; then
+        cat "$f" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o 'process.env.[A-Za-z0-9_]+' >> $TMPF
+    fi
 done
 
 cat $TMPF | sort -u | jq  --raw-input .  | jq --slurp .

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -5,7 +5,7 @@ fi
 
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-60}
+CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-120}
 tpid=""
 
 ctrl_c() {
@@ -20,6 +20,8 @@ if [ -z "$CERC_BUILD_TOOL" ]; then
     CERC_BUILD_TOOL=pnpm
   elif [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
+  elif [ -f "bun.lockb" ]; then
+    CERC_BUILD_TOOL=bun
   else
     CERC_BUILD_TOOL=npm
   fi
@@ -28,36 +30,44 @@ fi
 CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/app}"
 cd "$CERC_WEBAPP_FILES_DIR"
 
-"$SCRIPT_DIR/apply-runtime-env.sh" "`pwd`" .next .next-r
-mv .next .next.old
-mv .next-r/.next .
+if [ -f "./run-webapp.sh" ]; then
+  echo "Running webapp with run-webapp.sh ..."
+  cd "${WORK_DIR}" || exit 1
+  ./run-webapp.sh &
+  tpid=$!
+  wait $tpid
+else 
+  "$SCRIPT_DIR/apply-runtime-env.sh" "`pwd`" .next .next-r
+  mv .next .next.old
+  mv .next-r/.next .
 
-if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
-  jq -e '.scripts.cerc_generate' package.json >/dev/null
-  if [ $? -eq 0 ]; then
-    npm run cerc_generate > gen.out 2>&1 &
-    tail -f gen.out &
-    tpid=$!
+  if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
+    jq -e '.scripts.cerc_generate' package.json >/dev/null
+    if [ $? -eq 0 ]; then
+      npm run cerc_generate > gen.out 2>&1 &
+      tail -f gen.out &
+      tpid=$!
 
-    count=0
-    generate_done="false"
-    while [ $count -lt $CERC_MAX_GENERATE_TIME ] && [ "$generate_done" == "false" ]; do
-      sleep 1
-      count=$((count + 1))
-      grep 'rendered as static' gen.out > /dev/null
-      if [ $? -eq 0 ]; then
-        generate_done="true"
+      count=0
+      generate_done="false"
+      while [ $count -lt $CERC_MAX_GENERATE_TIME ] && [ "$generate_done" == "false" ]; do
+        sleep 1
+        count=$((count + 1))
+        grep 'rendered as static' gen.out > /dev/null
+        if [ $? -eq 0 ]; then
+          generate_done="true"
+        fi
+      done
+
+      if [ $generate_done != "true" ]; then
+        echo "ERROR: 'npm run cerc_generate' not successful within CERC_MAX_GENERATE_TIME" 1>&2
+        exit 1
       fi
-    done
 
-    if [ $generate_done != "true" ]; then
-      echo "ERROR: 'npm run cerc_generate' not successful within CERC_MAX_GENERATE_TIME" 1>&2
-      exit 1
+      kill $tpid $(ps -ef | grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
+      tpid=""
     fi
-
-    kill $tpid $(ps -ef | grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
-    tpid=""
   fi
-fi
 
-$CERC_BUILD_TOOL start . -- -p ${CERC_LISTEN_PORT:-80}
+  $CERC_BUILD_TOOL start . -- -p ${CERC_LISTEN_PORT:-80}
+fi

stack_orchestrator/data/container-build/cerc-ping-pub/Dockerfile

@@ -0,0 +1,10 @@
+FROM cerc/ping-pub-base:local
+
+COPY ./scripts/update-explorer-config.sh /scripts
+COPY ./scripts/start-serving-explorer.sh /scripts
+COPY ./config/laconic-chaindata-template.json /config/chains/laconic-chaindata-template.json
+
+EXPOSE 5173
+
+WORKDIR /app
+CMD ["/scripts/start-serving-explorer.sh"]

stack_orchestrator/data/container-build/cerc-ping-pub/Dockerfile.base

@@ -0,0 +1,8 @@
+FROM cerc/webapp-base:local
+
+WORKDIR /app
+
+COPY . .
+
+RUN yarn
+

stack_orchestrator/data/container-build/cerc-ping-pub/build.sh

@@ -1,5 +1,12 @@
 #!/usr/bin/env bash
 # Build the ping pub image
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-docker build -t cerc/ping-pub:local ${build_command_args} -f $CERC_REPO_BASE_DIR/explorer/Dockerfile $CERC_REPO_BASE_DIR/explorer
+# Two-stage build is to allow us to pick up both the upstream repo's files, and local files here for config
+docker build -t cerc/ping-pub-base:local ${build_command_args} -f $SCRIPT_DIR/Dockerfile.base $CERC_REPO_BASE_DIR/cosmos-explorer
+if [[ $? -ne 0 ]]; then
+    echo "FATAL: Base container build failed, exiting"
+    exit 1
+fi
+docker build -t cerc/ping-pub:local ${build_command_args} -f $SCRIPT_DIR/Dockerfile $SCRIPT_DIR

stack_orchestrator/data/container-build/cerc-ping-pub/config/laconic-chaindata-template.json

@@ -0,0 +1,22 @@
+{
+    "chain_name": "LACONIC_LACONICD_CHAIN_ID",
+    "registry_name": "LACONIC_LACONICD_CHAIN_ID",
+    "api": [
+        {"provider": "LX-one-tree-one-seven", "address": "LACONIC_LACONICD_API_URL"}
+    ],
+    "rpc": [
+        {"provider": "LX-tendermint-rpc", "address": "LACONIC_LACONICD_RPC_URL"}
+    ],
+    "sdk_version": "0.50.3",
+    "coin_type": "118",
+    "min_tx_fee": "800",
+    "addr_prefix": "laconic",
+    "logo": "/logos/cosmos.svg",
+    "assets": [{
+        "base": "alnt",
+        "symbol": "LNT",
+        "exponent": "18",
+        "coingecko_id": "cosmos",
+        "logo": "/logos/cosmos.svg"
+    }]
+}

stack_orchestrator/data/container-build/cerc-ping-pub/scripts/start-serving-explorer.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+${SCRIPT_DIR}/update-explorer-config.sh
+
+echo "Starting serving explorer"
+# Force cache re-build because vite is dumb and can't be restarted otherwise
+yarn serve --host --force

stack_orchestrator/data/container-build/cerc-ping-pub/scripts/update-explorer-config.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# Verify that we have the config variables we need
+if [[ -z ${LACONIC_LACONICD_API_URL} ]]; then
+    echo "Error: LACONIC_LACONICD_API_URL not defined"
+    exit 1
+fi
+if [[ -z ${LACONIC_LACONICD_RPC_URL} ]]; then
+    echo "Error: LACONIC_LACONICD_RPC_URL not defined"
+    exit 1
+fi
+if [[ -z ${LACONIC_LACONICD_CHAIN_ID} ]]; then
+    echo "Error: LACONIC_LACONICD_CHAIN_ID not defined"
+    exit 1
+fi
+
+# Ping-pub explorer has endlessly confusing behavior where it
+# infers the directory from which to load chain configuration files
+# by the presence or absense of the substring "testnet" in the host name
+# (browser side -- the host name of the host in the address bar of the browser)
+# Accordingly we configure our network in both directories in order to 
+# subvert this lunacy.
+explorer_mainnet_config_dir=/app/chains/mainnet
+explorer_testnet_config_dir=/app/chains/testnet
+config_template_file=/config/chains/laconic-chaindata-template.json
+chain_config_name=laconic.json
+mainnet_config_file=${explorer_mainnet_config_dir}/${chain_config_name}
+testnet_config_file=${explorer_testnet_config_dir}/${chain_config_name}
+
+# Delete the stock config files
+rm -f ${explorer_testnet_config_dir}/*
+rm -f ${explorer_mainnet_config_dir}/*
+
+# Copy in our template file
+cp ${config_template_file} ${mainnet_config_file}
+
+# Update the file with the config variables
+sed -i "s#LACONIC_LACONICD_API_URL#${LACONIC_LACONICD_API_URL}#g" ${mainnet_config_file}
+sed -i "s#LACONIC_LACONICD_RPC_URL#${LACONIC_LACONICD_RPC_URL}#g" ${mainnet_config_file}
+sed -i "s#LACONIC_LACONICD_CHAIN_ID#${LACONIC_LACONICD_CHAIN_ID}#g" ${mainnet_config_file}
+
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  echo "Updated chaindata file:"
+  cat ${mainnet_config_file}
+fi
+
+# Copy over to the testnet directory
+cp ${mainnet_config_file} ${testnet_config_file}

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/Dockerfile

@@ -1,6 +0,0 @@
-FROM cerc/snowballtools-base-backend-base:local
-
-WORKDIR /app/packages/backend
-COPY run.sh .
-
-ENTRYPOINT ["./run.sh"]

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/Dockerfile-base

@@ -1,26 +0,0 @@
-FROM ubuntu:22.04 as builder
-
-RUN apt update && \
-      apt install -y --no-install-recommends --no-install-suggests \
-      ca-certificates curl gnupg
-
-# Node
-ARG NODE_MAJOR=20
-RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
-      echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
-      apt update && apt install -y nodejs
-
-# npm setup
-RUN npm config set @cerc-io:registry https://git.vdb.to/api/packages/cerc-io/npm/ && npm install -g yarn
-
-COPY . /app/
-WORKDIR /app/
-
-RUN find . -name 'node_modules' | xargs -n1 rm -rf
-RUN yarn && yarn build --ignore frontend
-
-FROM cerc/webapp-base:local
-
-COPY --from=builder /app /app
-
-WORKDIR /app/packages/backend

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/build.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-# Build cerc/webapp-deployer-backend
-
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-
-# See: https://stackoverflow.com/a/246128/1701505
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-docker build -t cerc/snowballtools-base-backend-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile-base ${CERC_REPO_BASE_DIR}/snowballtools-base
-docker build -t cerc/snowballtools-base-backend:local ${build_command_args} ${SCRIPT_DIR} 

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/run.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-
-
-LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
-if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
-  if [ -f "/config/laconic-hosted-config.yml" ]; then
-    LACONIC_HOSTED_CONFIG_FILE="/config/laconic-hosted-config.yml"
-  elif [ -f "/config/config.yml" ]; then
-    LACONIC_HOSTED_CONFIG_FILE="/config/config.yml"
-  fi
-fi
-
-if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
-  /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "`pwd`/dist"
-fi
-
-/scripts/apply-runtime-env.sh "`pwd`/dist"
-
-yarn start

stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile

@@ -28,11 +28,13 @@ RUN \
     && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
     # Install pnpm
     && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
+    # Install bun
+    && su ${USERNAME} -c "umask 0002 && npm install -g bun@1.1.x" \
     && npm cache clean --force > /dev/null 2>&1
 
 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base
+    && apt-get -y install --no-install-recommends jq gettext-base git
 
 # [Optional] Uncomment if you want to install an additional version of node using nvm
 # ARG EXTRA_NODE_VERSION=10

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/apply-runtime-env.sh

@@ -4,6 +4,8 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
+# TODO: document what this script does
+
 WORK_DIR="${1:-./}"
 
 cd "${WORK_DIR}" || exit 1

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh

@@ -27,6 +27,8 @@ elif [ -f "${WORK_DIR}/package.json" ]; then
       CERC_BUILD_TOOL=pnpm
     elif [ -f "yarn.lock" ]; then
       CERC_BUILD_TOOL=yarn
+    elif [ -f "bun.lockb" ]; then
+      CERC_BUILD_TOOL=bun
     else
       CERC_BUILD_TOOL=npm
     fi

stack_orchestrator/data/npm-package-list.txt

@@ -1,4 +1,4 @@
-laconic-sdk
+registry-sdk
 laconic-registry-cli
 laconic-console
 debug

stack_orchestrator/data/repository-list.txt

@@ -7,7 +7,7 @@ github.com/cerc-io/ipld-eth-db-validator
 github.com/cerc-io/ipld-eth-beacon-indexer
 github.com/cerc-io/ipld-eth-beacon-db
 github.com/cerc-io/laconicd
-github.com/cerc-io/laconic-sdk
+github.com/cerc-io/registry-sdk
 github.com/cerc-io/laconic-registry-cli
 github.com/cerc-io/laconic-console
 github.com/cerc-io/mobymask-watcher-ts

stack_orchestrator/data/stacks/ajna/stack.yml

@@ -2,7 +2,7 @@ version: "1.0"
 name: ajna
 description: "Ajna watcher stack"
 repos:
-  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.11
+  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.13
 containers:
   - cerc/watcher-ajna
 pods:

stack_orchestrator/data/stacks/azimuth/README.md

@@ -44,34 +44,42 @@ network:
      - 0.0.0.0:9000:9000
     azimuth-watcher-server:
      - 0.0.0.0:3001:3001
+     - 0.0.0.0:9001:9001
     censures-watcher-job-runner:
      - 0.0.0.0:9002:9002
     censures-watcher-server:
      - 0.0.0.0:3002:3002
+     - 0.0.0.0:9003:9003
     claims-watcher-job-runner:
      - 0.0.0.0:9004:9004
     claims-watcher-server:
      - 0.0.0.0:3003:3003
+     - 0.0.0.0:9005:9005
     conditional-star-release-watcher-job-runner:
      - 0.0.0.0:9006:9006
     conditional-star-release-watcher-server:
      - 0.0.0.0:3004:3004
+     - 0.0.0.0:9007:9007
     delegated-sending-watcher-job-runner:
      - 0.0.0.0:9008:9008
     delegated-sending-watcher-server:
      - 0.0.0.0:3005:3005
+     - 0.0.0.0:9009:9009
     ecliptic-watcher-job-runner:
      - 0.0.0.0:9010:9010
     ecliptic-watcher-server:
      - 0.0.0.0:3006:3006
+     - 0.0.0.0:9011:9011
     linear-star-release-watcher-job-runner:
      - 0.0.0.0:9012:9012
     linear-star-release-watcher-server:
      - 0.0.0.0:3007:3007
+     - 0.0.0.0:9013:9013
     polls-watcher-job-runner:
      - 0.0.0.0:9014:9014
     polls-watcher-server:
      - 0.0.0.0:3008:3008
+     - 0.0.0.0:9015:9015
     gateway-server:
      - 0.0.0.0:4000:4000
 ...

stack_orchestrator/data/stacks/azimuth/stack.yml

@@ -1,7 +1,7 @@
 version: "1.0"
 name: azimuth
 repos:
-  - github.com/cerc-io/azimuth-watcher-ts@0.1.5
+  - github.com/cerc-io/azimuth-watcher-ts@0.1.6
 containers:
   - cerc/watcher-azimuth
 pods:

stack_orchestrator/data/stacks/build-support/README.md

@@ -58,5 +58,5 @@ Now npm packages can be built:
 Ensure that `CERC_NPM_AUTH_TOKEN` is set with the token printed above when the package-registry stack was deployed (the actual token value will be different than shown in this example):
 ```
 $ export CERC_NPM_AUTH_TOKEN=84fe66a73698bf11edbdccd0a338236b7d1d5c45
-$ laconic-so build-npms --include laconic-sdk,laconic-registry-cli
+$ laconic-so build-npms --include registry-sdk,laconic-registry-cli
 ```

stack_orchestrator/data/stacks/fixturenet-laconic-loaded/README.md

@@ -49,7 +49,7 @@ $ laconic-so --stack fixturenet-laconic-loaded deploy logs
 ```
 ## 6. Test with the Registry CLI
 ```
-$ laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns status"
+$ laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic registry status"
 ```
 ## 7. View the laconic console
 Get the URL for the console web app with this command (the port number will be different for each deployment):
@@ -64,5 +64,6 @@ $ laconic-so --stack fixturenet-laconic-loaded deploy exec cli ./scripts/create-
 Balance is: 99998999999999998999600000
 Created bond with id: dd88e8d6f9567b32b28e70552aea4419c5dd3307ebae85a284d1fe38904e301a
 Published demo-record-1.yml with id: bafyreierh3xnfivexlscdwubvczmddsnf46uytyfvrbdhkjzztvsz6ruly
+...
 ```
-The published record should be visible in the console.
+The published records should be visible in the console.

stack_orchestrator/data/stacks/fixturenet-laconic-loaded/stack.yml

@@ -7,11 +7,11 @@ repos:
   - github.com/lirewine/crypto
   - github.com/lirewine/gem
   - github.com/lirewine/sdk
-  - git.vdb.to/cerc-io/laconic-sdk
+  - git.vdb.to/cerc-io/registry-sdk
   - git.vdb.to/cerc-io/laconic-registry-cli
   - git.vdb.to/cerc-io/laconic-console
 npms:
-  - laconic-sdk
+  - registry-sdk
   - laconic-registry-cli
   - debug
   - crypto
@@ -30,4 +30,3 @@ config:
   cli:
     key: laconicd.mykey
     address: laconicd.myaddress
-

stack_orchestrator/data/stacks/fixturenet-laconicd/README.md

@@ -44,5 +44,5 @@ $ laconic-so --stack fixturenet-laconicd deploy logs
 ```
 ## 6. Test with the Registry CLI
 ```
-$ laconic-so --stack fixturenet-laconicd deploy exec cli "laconic cns status"
+$ laconic-so --stack fixturenet-laconicd deploy exec cli "laconic registry status"
 ```

stack_orchestrator/data/stacks/fixturenet-laconicd/stack.yml

@@ -3,10 +3,10 @@ name: fixturenet-laconicd
 description: "A laconicd fixturenet"
 repos:
   - git.vdb.to/cerc-io/laconicd
-  - git.vdb.to/cerc-io/laconic-sdk
+  - git.vdb.to/cerc-io/registry-sdk
   - git.vdb.to/cerc-io/laconic-registry-cli
 npms:
-  - laconic-sdk
+  - registry-sdk
   - laconic-registry-cli
 containers:
   - cerc/laconicd

stack_orchestrator/data/stacks/mainnet-laconic/deploy/commands.py

@@ -14,14 +14,14 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from stack_orchestrator.util import get_yaml
-from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
+from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand
+from stack_orchestrator.deploy.deployment_context import DeploymentContext
 from stack_orchestrator.deploy.stack_state import State
 from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command
-from stack_orchestrator.command_types import CommandOptions
+from stack_orchestrator.opts import opts
 from enum import Enum
 from pathlib import Path
 from shutil import copyfile, copytree
-import json
 import os
 import sys
 import tomli
@@ -33,8 +33,9 @@ default_spec_file_content = ""
 class SetupPhase(Enum):
     INITIALIZE = 1
     JOIN = 2
-    CREATE = 3
-    ILLEGAL = 3
+    CONNECT = 3
+    CREATE = 4
+    ILLEGAL = 5
 
 
 def _client_toml_path(network_dir: Path):
@@ -61,39 +62,23 @@ def _get_node_moniker_from_config(network_dir: Path):
     return moniker
 
 
-def _get_node_key_from_gentx(options: CommandOptions, gentx_file_name: str):
-    gentx_file_path = Path(gentx_file_name)
-    if gentx_file_path.exists():
-        with open(Path(gentx_file_name), "rb") as f:
-            parsed_json = json.load(f)
-            return parsed_json['body']['messages'][0]['delegator_address']
-    else:
-        print(f"Error: gentx file: {gentx_file_name} does not exist")
-        sys.exit(1)
-
-
 def _comma_delimited_to_list(list_str: str):
     return list_str.split(",") if list_str else []
 
 
-def _get_node_keys_from_gentx_files(options: CommandOptions, gentx_file_list: str):
-    node_keys = []
-    gentx_files = _comma_delimited_to_list(gentx_file_list)
-    for gentx_file in gentx_files:
-        node_key = _get_node_key_from_gentx(options, gentx_file)
-        if node_key:
-            node_keys.append(node_key)
-    return node_keys
+def _get_node_keys_from_gentx_files(gentx_address_list: str):
+    gentx_addresses = _comma_delimited_to_list(gentx_address_list)
+    return gentx_addresses
 
 
-def _copy_gentx_files(options: CommandOptions, network_dir: Path, gentx_file_list: str):
+def _copy_gentx_files(network_dir: Path, gentx_file_list: str):
     gentx_files = _comma_delimited_to_list(gentx_file_list)
     for gentx_file in gentx_files:
         gentx_file_path = Path(gentx_file)
         copyfile(gentx_file_path, os.path.join(network_dir, "config", "gentx", os.path.basename(gentx_file_path)))
 
 
-def _remove_persistent_peers(options: CommandOptions, network_dir: Path):
+def _remove_persistent_peers(network_dir: Path):
     config_file_path = _config_toml_path(network_dir)
     if not config_file_path.exists():
         print("Error: config.toml not found")
@@ -107,20 +92,74 @@ def _remove_persistent_peers(options: CommandOptions, network_dir: Path):
         output_file.write(config_file_content)
 
 
-def _insert_persistent_peers(options: CommandOptions, config_dir: Path, new_persistent_peers: str):
+def _insert_persistent_peers(config_dir: Path, new_persistent_peers: str):
     config_file_path = config_dir.joinpath("config.toml")
     if not config_file_path.exists():
         print("Error: config.toml not found")
         sys.exit(1)
     with open(config_file_path, "r") as input_file:
         config_file_content = input_file.read()
-        persistent_peers_pattern = '^persistent_peers = ""'
+        persistent_peers_pattern = r'^persistent_peers = ""'
         replace_with = f"persistent_peers = \"{new_persistent_peers}\""
         config_file_content = re.sub(persistent_peers_pattern, replace_with, config_file_content, flags=re.MULTILINE)
     with open(config_file_path, "w") as output_file:
         output_file.write(config_file_content)
 
 
+def _enable_cors(config_dir: Path):
+    config_file_path = config_dir.joinpath("config.toml")
+    if not config_file_path.exists():
+        print("Error: config.toml not found")
+        sys.exit(1)
+    with open(config_file_path, "r") as input_file:
+        config_file_content = input_file.read()
+        cors_pattern = r'^cors_allowed_origins = \[]'
+        replace_with = 'cors_allowed_origins = ["*"]'
+        config_file_content = re.sub(cors_pattern, replace_with, config_file_content, flags=re.MULTILINE)
+    with open(config_file_path, "w") as output_file:
+        output_file.write(config_file_content)
+    app_file_path = config_dir.joinpath("app.toml")
+    if not app_file_path.exists():
+        print("Error: app.toml not found")
+        sys.exit(1)
+    with open(app_file_path, "r") as input_file:
+        app_file_content = input_file.read()
+        cors_pattern = r'^enabled-unsafe-cors = false'
+        replace_with = "enabled-unsafe-cors = true"
+        app_file_content = re.sub(cors_pattern, replace_with, app_file_content, flags=re.MULTILINE)
+    with open(app_file_path, "w") as output_file:
+        output_file.write(app_file_content)
+
+
+def _set_listen_address(config_dir: Path):
+    config_file_path = config_dir.joinpath("config.toml")
+    if not config_file_path.exists():
+        print("Error: config.toml not found")
+        sys.exit(1)
+    with open(config_file_path, "r") as input_file:
+        config_file_content = input_file.read()
+        existing_pattern = r'^laddr = "tcp://127.0.0.1:26657"'
+        replace_with = 'laddr = "tcp://0.0.0.0:26657"'
+        print(f"Replacing in: {config_file_path}")
+        config_file_content = re.sub(existing_pattern, replace_with, config_file_content, flags=re.MULTILINE)
+    with open(config_file_path, "w") as output_file:
+        output_file.write(config_file_content)
+    app_file_path = config_dir.joinpath("app.toml")
+    if not app_file_path.exists():
+        print("Error: app.toml not found")
+        sys.exit(1)
+    with open(app_file_path, "r") as input_file:
+        app_file_content = input_file.read()
+        existing_pattern1 = r'^address = "tcp://localhost:1317"'
+        replace_with1 = 'address = "tcp://0.0.0.0:1317"'
+        app_file_content = re.sub(existing_pattern1, replace_with1, app_file_content, flags=re.MULTILINE)
+        existing_pattern2 = r'^address = "localhost:9090"'
+        replace_with2 = 'address = "0.0.0.0:9090"'
+        app_file_content = re.sub(existing_pattern2, replace_with2, app_file_content, flags=re.MULTILINE)
+    with open(app_file_path, "w") as output_file:
+        output_file.write(app_file_content)
+
+
 def _phase_from_params(parameters):
     phase = SetupPhase.ILLEGAL
     if parameters.initialize_network:
@@ -145,14 +184,19 @@ def _phase_from_params(parameters):
             print("Can't supply --initialize-network or --join-network with --create-network")
             sys.exit(1)
         phase = SetupPhase.CREATE
+    elif parameters.connect_network:
+        if parameters.initialize_network or parameters.join_network:
+            print("Can't supply --initialize-network or --join-network with --connect-network")
+            sys.exit(1)
+        phase = SetupPhase.CONNECT
     return phase
 
 
 def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCommand, extra_args):
 
-    options = command_context.cluster_context.options
+    options = opts.o
 
-    currency = "stake"  # Does this need to be a parameter?
+    currency = "alnt"  # Does this need to be a parameter?
 
     if options.debug:
         print(f"parameters: {parameters}")
@@ -177,11 +221,12 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
         output, status = run_container_command(
             command_context,
             "laconicd", f"laconicd init {parameters.node_moniker} --home {laconicd_home_path_in_container}\
-                --chain-id {parameters.chain_id}", mounts)
+                --chain-id {parameters.chain_id} --default-denom {currency}", mounts)
         if options.debug:
             print(f"Command output: {output}")
 
     elif phase == SetupPhase.JOIN:
+        # In the join phase (alternative to connect) we are participating in a genesis ceremony for the chain
         if not os.path.exists(network_dir):
             print(f"Error: network directory {network_dir} doesn't exist")
             sys.exit(1)
@@ -196,7 +241,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
         output2, status2 = run_container_command(
             command_context,
             "laconicd",
-            f"laconicd add-genesis-account {parameters.key_name} 12900000000000000000000{currency}\
+            f"laconicd genesis add-genesis-account {parameters.key_name} 12900000000000000000000{currency}\
                 --home {laconicd_home_path_in_container} --keyring-backend test",
             mounts)
         if options.debug:
@@ -204,7 +249,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
         output3, status3 = run_container_command(
             command_context,
             "laconicd",
-            f"laconicd gentx  {parameters.key_name} 90000000000{currency} --home {laconicd_home_path_in_container}\
+            f"laconicd genesis gentx  {parameters.key_name} 90000000000{currency} --home {laconicd_home_path_in_container}\
                 --chain-id {chain_id} --keyring-backend test",
             mounts)
         if options.debug:
@@ -214,7 +259,28 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
             "laconicd",
             f"laconicd keys show  {parameters.key_name} -a --home {laconicd_home_path_in_container} --keyring-backend test",
             mounts)
-        print(f"Node validator address: {output4}")
+        print(f"Node account address: {output4}")
+
+    elif phase == SetupPhase.CONNECT:
+        # In the connect phase (named to not conflict with join) we are making a node that syncs a chain with existing genesis.json
+        # but not with validator role. We need this kind of node in order to bootstrap it into a validator after it syncs
+        output1, status1 = run_container_command(
+            command_context, "laconicd", f"laconicd keys add {parameters.key_name} --home {laconicd_home_path_in_container}\
+                --keyring-backend test", mounts)
+        if options.debug:
+            print(f"Command output: {output1}")
+        output2, status2 = run_container_command(
+            command_context,
+            "laconicd",
+            f"laconicd keys show  {parameters.key_name} -a --home {laconicd_home_path_in_container} --keyring-backend test",
+            mounts)
+        print(f"Node account address: {output2}")
+        output3, status3 = run_container_command(
+            command_context,
+            "laconicd",
+            f"laconicd cometbft show-validator --home {laconicd_home_path_in_container}",
+            mounts)
+        print(f"Node validator address: {output3}")
 
     elif phase == SetupPhase.CREATE:
         if not os.path.exists(network_dir):
@@ -233,32 +299,30 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
             copyfile(genesis_file_path, os.path.join(network_dir, "config", os.path.basename(genesis_file_path)))
         else:
             # We're generating the genesis file
-            if not parameters.gentx_file_list:
-                print("Error: --gentx-files must be supplied")
-                sys.exit(1)
             # First look in the supplied gentx files for the other nodes' keys
-            other_node_keys = _get_node_keys_from_gentx_files(options, parameters.gentx_file_list)
+            other_node_keys = _get_node_keys_from_gentx_files(parameters.gentx_address_list)
             # Add those keys to our genesis, with balances we determine here (why?)
             for other_node_key in other_node_keys:
                 outputk, statusk = run_container_command(
-                    command_context, "laconicd", f"laconicd add-genesis-account {other_node_key} 12900000000000000000000{currency}\
+                    command_context, "laconicd", f"laconicd genesis add-genesis-account {other_node_key} \
+                        12900000000000000000000{currency}\
                         --home {laconicd_home_path_in_container} --keyring-backend test", mounts)
             if options.debug:
                 print(f"Command output: {outputk}")
             # Copy the gentx json files into our network dir
-            _copy_gentx_files(options, network_dir, parameters.gentx_file_list)
+            _copy_gentx_files(network_dir, parameters.gentx_file_list)
             # Now we can run collect-gentxs
             output1, status1 = run_container_command(
-                command_context, "laconicd", f"laconicd collect-gentxs --home {laconicd_home_path_in_container}", mounts)
+                command_context, "laconicd", f"laconicd genesis collect-gentxs --home {laconicd_home_path_in_container}", mounts)
             if options.debug:
                 print(f"Command output: {output1}")
             print(f"Generated genesis file, please copy to other nodes as required: \
                 {os.path.join(network_dir, 'config', 'genesis.json')}")
             # Last thing, collect-gentxs puts a likely bogus set of persistent_peers in config.toml so we remove that now
-            _remove_persistent_peers(options, network_dir)
+            _remove_persistent_peers(network_dir)
         # In both cases we validate the genesis file now
         output2, status1 = run_container_command(
-            command_context, "laconicd", f"laconicd validate-genesis --home {laconicd_home_path_in_container}", mounts)
+            command_context, "laconicd", f"laconicd genesis validate-genesis --home {laconicd_home_path_in_container}", mounts)
         print(f"validate-genesis result: {output2}")
 
     else:
@@ -266,7 +330,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
         sys.exit(1)
 
 
-def create(context: DeploymentContext, extra_args):
+def create(deployment_context: DeploymentContext, extra_args):
     network_dir = extra_args[0]
     if network_dir is None:
         print("Error: --network-dir must be supplied")
@@ -285,15 +349,18 @@ def create(context: DeploymentContext, extra_args):
         sys.exit(1)
     # Copy the network directory contents into our deployment
     # TODO: change this to work with non local paths
-    deployment_config_dir = context.deployment_dir.joinpath("data", "laconicd-config")
+    deployment_config_dir = deployment_context.deployment_dir.joinpath("data", "laconicd-config")
     copytree(config_dir_path, deployment_config_dir, dirs_exist_ok=True)
     # If supplied, add the initial persistent peers to the config file
     if extra_args[1]:
         initial_persistent_peers = extra_args[1]
-        _insert_persistent_peers(context.command_context.cluster_context.options, deployment_config_dir, initial_persistent_peers)
+        _insert_persistent_peers(deployment_config_dir, initial_persistent_peers)
+    # Enable CORS headers so explorers and so on can talk to the node
+    _enable_cors(deployment_config_dir)
+    _set_listen_address(deployment_config_dir)
     # Copy the data directory contents into our deployment
     # TODO: change this to work with non local paths
-    deployment_data_dir = context.deployment_dir.joinpath("data", "laconicd-data")
+    deployment_data_dir = deployment_context.deployment_dir.joinpath("data", "laconicd-data")
     copytree(data_dir_path, deployment_data_dir, dirs_exist_ok=True)
 
 
@@ -303,7 +370,6 @@ def init(command_context: DeployCommandContext):
 
 
 def get_state(command_context: DeployCommandContext):
-    print("Here we get state")
     return State.CONFIGURED
 
 

stack_orchestrator/data/stacks/mainnet-laconic/stack.yml

@@ -2,16 +2,17 @@ version: "1.0"
 name: mainnet-laconic
 description: "Mainnet laconic node"
 repos:
-  - cerc-io/laconicd
-  - lirewine/debug
-  - lirewine/crypto
-  - lirewine/gem
-  - lirewine/sdk
-  - cerc-io/laconic-sdk
-  - cerc-io/laconic-registry-cli
-  - cerc-io/laconic-console
+  - git.vdb.to/cerc-io/laconicd
+  - github.com/lirewine/debug
+  - github.com/lirewine/crypto
+  - github.com/lirewine/gem
+  - github.com/lirewine/sdk
+  - git.vdb.to/cerc-io/registry-sdk
+  - git.vdb.to/cerc-io/laconic-registry-cli
+  - git.vdb.to/cerc-io/laconic-console
+  - git.vdb.to/cerc-io/cosmos-explorer
 npms:
-  - laconic-sdk
+  - registry-sdk
   - laconic-registry-cli
   - debug
   - crypto
@@ -23,7 +24,8 @@ containers:
   - cerc/laconic-registry-cli
   - cerc/webapp-base
   - cerc/laconic-console-host
+  - cerc/ping-pub
 pods:
   - mainnet-laconicd
   - fixturenet-laconic-console
-
+  - laconic-explorer

stack_orchestrator/data/stacks/mainnet-laconic/test/run-mainnet-laconic-test.sh

@@ -8,8 +8,11 @@ echo "Environment variables:"
 env
 # Test laconic stack
 echo "Running laconic stack test"
-# Bit of a hack, test the most recent package
-TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+if [ "$1" == "from-path" ]; then
+    TEST_TARGET_SO="laconic-so"
+else
+    TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+fi
 # Set a non-default repo dir
 export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
 echo "Testing this package: $TEST_TARGET_SO"

stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml

@@ -2,7 +2,7 @@ version: "1.0"
 name: merkl-sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.12
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.14
 containers:
   - cerc/watcher-merkl-sushiswap-v3
 pods:

stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md

@@ -57,35 +57,35 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
     metrics_path: /metrics
     scheme: http
     static_configs:
-      - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_PORT']
+      - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_METRICS_PORT', 'AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'azimuth'
           chain: 'ethereum'
-      - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_PORT']
+      - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_METRICS_PORT', 'CENSURES_WATCHER_HOST:CENSURES_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'censures'
           chain: 'ethereum'
-      - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_PORT']
+      - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_METRICS_PORT', 'CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'claims'
           chain: 'ethereum'
-      - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_PORT']
+      - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_METRICS_PORT', 'CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'conditional_star_release'
           chain: 'ethereum'
-      - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_PORT']
+      - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_METRICS_PORT', 'DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'delegated_sending'
           chain: 'ethereum'
-      - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_PORT']
+      - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_METRICS_PORT', 'ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'ecliptic'
           chain: 'ethereum'
-      - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_PORT']
+      - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_METRICS_PORT', 'LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'linear_star_release'
           chain: 'ethereum'
-      - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_PORT']
+      - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_METRICS_PORT', 'POLLS_WATCHER_HOST:POLLS_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'polls'
           chain: 'ethereum'
@@ -95,11 +95,11 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
     metrics_path: /metrics
     scheme: http
     static_configs:
-      - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_PORT']
+      - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_METRICS_PORT', 'SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'sushiswap'
           chain: 'filecoin'
-      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_PORT']
+      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_METRICS_PORT', 'MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'merkl_sushiswap'
           chain: 'filecoin'
@@ -109,7 +109,7 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
     metrics_path: /metrics
     scheme: http
     static_configs:
-      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_PORT']
+      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_METRICS_PORT', 'AJNA_WATCHER_HOST:AJNA_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'ajna'
           chain: 'filecoin'

stack_orchestrator/data/stacks/osmosis/README.md

@@ -109,7 +109,7 @@ Setup a test chain:
 ```bash
 export CERC_NPM_REGISTRY_URL=https://git.vdb.to/api/packages/cerc-io/npm/
 
-laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
+laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/registry-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
 
 laconic-so --stack fixturenet-laconic-loaded build-containers
 

stack_orchestrator/data/stacks/package-registry/README.md

@@ -2,4 +2,50 @@
 
 The Package Registry Stack supports a build environment that requires a package registry (initially for NPM packages only).
 
-Setup instructions can be found [here](../build-support/README.md).
+## Setup
+
+* Setup required repos and build containers:
+
+  ```bash
+  laconic-so --stack package-registry setup-repositories
+  laconic-so --stack package-registry build-containers
+  ```
+
+* Create a deployment:
+
+  ```bash
+  laconic-so --stack package-registry deploy init --output package-registry-spec.yml
+  # Update port mapping in the laconic-loaded.spec file to resolve port conflicts on host if any
+
+  laconic-so --stack package-registry deploy create --deployment-dir package-registry-deployment --spec-file package-registry-spec.yml
+  ```
+
+* Start the deployment:
+
+  ```bash
+  laconic-so deployment --dir package-registry-deployment start
+  ```
+
+* The local gitea registry can now be accessed at <http://localhost:3000> (the username and password can be taken from the deployment logs)
+
+* Configure the hostname `gitea.local`:
+
+  Update `/etc/hosts`:
+
+  ```bash
+  sudo nano /etc/hosts
+
+  # Add the following line
+  127.0.0.1       gitea.local
+  ```
+
+  Check resolution:
+
+  ```bash
+  ping gitea.local
+
+  PING gitea.local (127.0.0.1) 56(84) bytes of data.
+  64 bytes from localhost (127.0.0.1): icmp_seq=1 ttl=64 time=0.147 ms
+  64 bytes from localhost (127.0.0.1): icmp_seq=2 ttl=64 time=0.033 ms
+  ...
+  ```

stack_orchestrator/data/stacks/snowballtools-base-backend/stack.yml

@@ -1,10 +0,0 @@
-version: "1.0"
-name: snowballtools-base-backend
-description: "snowballtools-base-backend"
-repos:
-  - github.com/snowball-tools/snowballtools-base
-containers:
-  - cerc/webapp-base
-  - cerc/snowballtools-base-backend
-pods:
-  - snowballtools-base-backend

stack_orchestrator/data/stacks/sushiswap-v3/stack.yml

@@ -2,7 +2,7 @@ version: "1.0"
 name: sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.12
+  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.14
 containers:
   - cerc/watcher-sushiswap-v3
 pods:

stack_orchestrator/deploy/compose/deploy_docker.py

@@ -29,14 +29,14 @@ class DockerDeployer(Deployer):
                                    compose_env_file=compose_env_file)
         self.type = type
 
-    def up(self, detach, services):
+    def up(self, detach, skip_cluster_management, services):
         if not opts.o.dry_run:
             try:
                 return self.docker.compose.up(detach=detach, services=services)
             except DockerException as e:
                 raise DeployerException(e)
 
-    def down(self, timeout, volumes):
+    def down(self, timeout, volumes, skip_cluster_management):
         if not opts.o.dry_run:
             try:
                 return self.docker.compose.down(timeout=timeout, volumes=volumes)
@@ -94,6 +94,40 @@ class DockerDeployer(Deployer):
             except DockerException as e:
                 raise DeployerException(e)
 
+    def run_job(self, job_name: str, release_name: str = None):
+        # release_name is ignored for Docker deployments (only used for K8s/Helm)
+        if not opts.o.dry_run:
+            try:
+                # Find job compose file in compose-jobs directory
+                # The deployment should have compose-jobs/docker-compose-<job_name>.yml
+                if not self.docker.compose_files:
+                    raise DeployerException("No compose files configured")
+
+                # Deployment directory is parent of compose directory
+                compose_dir = Path(self.docker.compose_files[0]).parent
+                deployment_dir = compose_dir.parent
+                job_compose_file = deployment_dir / "compose-jobs" / f"docker-compose-{job_name}.yml"
+
+                if not job_compose_file.exists():
+                    raise DeployerException(f"Job compose file not found: {job_compose_file}")
+
+                if opts.o.verbose:
+                    print(f"Running job from: {job_compose_file}")
+
+                # Create a DockerClient for the job compose file with same project name and env file
+                # This allows the job to access volumes from the main deployment
+                job_docker = DockerClient(
+                    compose_files=[job_compose_file],
+                    compose_project_name=self.docker.compose_project_name,
+                    compose_env_file=self.docker.compose_env_file
+                )
+
+                # Run the job with --rm flag to remove container after completion
+                return job_docker.compose.run(service=job_name, remove=True, tty=True)
+
+            except DockerException as e:
+                raise DeployerException(e)
+
 
 class DockerDeployerConfigGenerator(DeployerConfigGenerator):
 

stack_orchestrator/deploy/deploy.py

@@ -26,8 +26,15 @@ import click
 from pathlib import Path
 from stack_orchestrator import constants
 from stack_orchestrator.opts import opts
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path
-from stack_orchestrator.util import resolve_compose_file
+from stack_orchestrator.util import (
+    get_stack_path,
+    include_exclude_check,
+    get_parsed_stack_config,
+    global_options2,
+    get_dev_root_path,
+    stack_is_in_deployment,
+    resolve_compose_file,
+)
 from stack_orchestrator.deploy.deployer import Deployer, DeployerException
 from stack_orchestrator.deploy.deployer_factory import getDeployer
 from stack_orchestrator.deploy.deploy_types import ClusterContext, DeployCommandContext
@@ -60,6 +67,7 @@ def command(ctx, include, exclude, env_file, cluster, deploy_to):
     if deploy_to is None:
         deploy_to = "compose"
 
+    stack = get_stack_path(stack)
     ctx.obj = create_deploy_context(global_options2(ctx), None, stack, include, exclude, cluster, env_file, deploy_to)
     # Subcommand is executed now, by the magic of click
 
@@ -76,14 +84,29 @@ def create_deploy_context(
     # Extract the cluster name from the deployment, if we have one
     if deployment_context and cluster is None:
         cluster = deployment_context.get_cluster_id()
-    cluster_context = _make_cluster_context(global_context, stack, include, exclude, cluster, env_file)
+
+    # Check if this is a helm chart deployment (has chart/ but no compose/)
+    # TODO: Add a new deployment type for helm chart deployments
+    # To avoid relying on chart existence in such cases
+    is_helm_chart_deployment = False
+    if deployment_context:
+        chart_dir = deployment_context.deployment_dir / "chart"
+        compose_dir = deployment_context.deployment_dir / "compose"
+        is_helm_chart_deployment = chart_dir.exists() and not compose_dir.exists()
+
+    # For helm chart deployments, skip compose file loading
+    if is_helm_chart_deployment:
+        cluster_context = ClusterContext(global_context, cluster, [], [], [], None, env_file)
+    else:
+        cluster_context = _make_cluster_context(global_context, stack, include, exclude, cluster, env_file)
+
     deployer = getDeployer(deploy_to, deployment_context, compose_files=cluster_context.compose_files,
                            compose_project_name=cluster_context.cluster,
                            compose_env_file=cluster_context.env_file)
     return DeployCommandContext(stack, cluster_context, deployer)
 
 
-def up_operation(ctx, services_list, stay_attached=False):
+def up_operation(ctx, services_list, stay_attached=False, skip_cluster_management=False):
     global_context = ctx.parent.parent.obj
     deploy_context = ctx.obj
     cluster_context = deploy_context.cluster_context
@@ -94,18 +117,18 @@ def up_operation(ctx, services_list, stay_attached=False):
         print(f"Running compose up with container_exec_env: {container_exec_env}, extra_args: {services_list}")
     for pre_start_command in cluster_context.pre_start_commands:
         _run_command(global_context, cluster_context.cluster, pre_start_command)
-    deploy_context.deployer.up(detach=not stay_attached, services=services_list)
+    deploy_context.deployer.up(detach=not stay_attached, skip_cluster_management=skip_cluster_management, services=services_list)
     for post_start_command in cluster_context.post_start_commands:
         _run_command(global_context, cluster_context.cluster, post_start_command)
     _orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.deployer, container_exec_env)
 
 
-def down_operation(ctx, delete_volumes, extra_args_list):
+def down_operation(ctx, delete_volumes, extra_args_list, skip_cluster_management=False):
     timeout_arg = None
     if extra_args_list:
         timeout_arg = extra_args_list[0]
     # Specify shutdown timeout (default 10s) to give services enough time to shutdown gracefully
-    ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes)
+    ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes, skip_cluster_management=skip_cluster_management)
 
 
 def status_operation(ctx):
@@ -180,6 +203,17 @@ def logs_operation(ctx, tail: int, follow: bool, extra_args: str):
         print(stream_content.decode("utf-8"), end="")
 
 
+def run_job_operation(ctx, job_name: str, helm_release: str = None):
+    global_context = ctx.parent.parent.obj
+    if not global_context.dry_run:
+        print(f"Running job: {job_name}")
+        try:
+            ctx.obj.deployer.run_job(job_name, helm_release)
+        except Exception as e:
+            print(f"Error running job {job_name}: {e}")
+            sys.exit(1)
+
+
 @command.command()
 @click.argument('extra_args', nargs=-1)  # help: command: up <service1> <service2>
 @click.pass_context
@@ -274,16 +308,12 @@ def _make_default_cluster_name(deployment, compose_dir, stack, include, exclude)
 
 # stack has to be either PathLike pointing to a stack yml file, or a string with the name of a known stack
 def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
-
     dev_root_path = get_dev_root_path(ctx)
 
-    # TODO: huge hack, fix this
-    # If the caller passed a path for the stack file, then we know that we can get the compose files
-    # from the same directory
-    deployment = False
-    if isinstance(stack, os.PathLike):
-        compose_dir = stack.parent.joinpath("compose")
-        deployment = True
+    # TODO: hack, this should be encapsulated by the deployment context.
+    deployment = stack_is_in_deployment(stack)
+    if deployment:
+        compose_dir = stack.joinpath("compose")
     else:
         # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
         compose_dir = Path(__file__).absolute().parent.parent.joinpath("data", "compose")

stack_orchestrator/deploy/deploy_types.py

@@ -50,8 +50,10 @@ class LaconicStackSetupCommand:
     key_name: str
     initialize_network: bool
     join_network: bool
+    connect_network: bool
     create_network: bool
     gentx_file_list: str
+    gentx_address_list: str
     genesis_file: str
     network_dir: str
 

stack_orchestrator/deploy/deploy_util.py

@@ -13,7 +13,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-import os
 from typing import List, Any
 from stack_orchestrator.deploy.deploy_types import DeployCommandContext, VolumeMapping
 from stack_orchestrator.util import get_parsed_stack_config, get_yaml, get_pod_list, resolve_compose_file
@@ -83,7 +82,9 @@ def run_container_command(ctx: DeployCommandContext, service: str, command: str,
     docker_output = deployer.run(
         container_image,
         ["-c", command], entrypoint="sh",
-        user=f"{os.getuid()}:{os.getgid()}",
+        # Current laconicd container has a bug where it crashes when run not as root
+        # Commented out line below is a workaround. Created files end up owned by root on the host
+        # user=f"{os.getuid()}:{os.getgid()}",
         volumes=docker_volumes
         )
     # There doesn't seem to be a way to get an exit code from docker.run()

stack_orchestrator/deploy/deployer.py

@@ -20,11 +20,11 @@ from pathlib import Path
 class Deployer(ABC):
 
     @abstractmethod
-    def up(self, detach, services):
+    def up(self, detach, skip_cluster_management, services):
         pass
 
     @abstractmethod
-    def down(self, timeout, volumes):
+    def down(self, timeout, volumes, skip_cluster_management):
         pass
 
     @abstractmethod
@@ -55,6 +55,10 @@ class Deployer(ABC):
     def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False):
         pass
 
+    @abstractmethod
+    def run_job(self, job_name: str, release_name: str = None):
+        pass
+
 
 class DeployerException(Exception):
     def __init__(self, *args: object) -> None:

stack_orchestrator/deploy/deployment.py

@@ -50,58 +50,68 @@ def command(ctx, dir):
 
 def make_deploy_context(ctx) -> DeployCommandContext:
     context: DeploymentContext = ctx.obj
-    stack_file_path = context.get_stack_file()
     env_file = context.get_env_file()
     cluster_name = context.get_cluster_id()
     if constants.deploy_to_key in context.spec.obj:
         deployment_type = context.spec.obj[constants.deploy_to_key]
     else:
         deployment_type = constants.compose_deploy_type
-    return create_deploy_context(ctx.parent.parent.obj, context, stack_file_path, None, None, cluster_name, env_file,
-                                 deployment_type)
+    stack = context.deployment_dir
+    return create_deploy_context(ctx.parent.parent.obj, context, stack, None, None,
+                                 cluster_name, env_file, deployment_type)
 
 
+# TODO: remove legacy up command since it's an alias for start
 @command.command()
 @click.option("--stay-attached/--detatch-terminal", default=False, help="detatch or not to see container stdout")
+@click.option("--skip-cluster-management/--perform-cluster-management",
+              default=False, help="Skip cluster initialization/tear-down (only for kind-k8s deployments)")
 @click.argument('extra_args', nargs=-1)  # help: command: up <service1> <service2>
 @click.pass_context
-def up(ctx, stay_attached, extra_args):
+def up(ctx, stay_attached, skip_cluster_management, extra_args):
     ctx.obj = make_deploy_context(ctx)
     services_list = list(extra_args) or None
-    up_operation(ctx, services_list, stay_attached)
+    up_operation(ctx, services_list, stay_attached, skip_cluster_management)
 
 
 # start is the preferred alias for up
 @command.command()
 @click.option("--stay-attached/--detatch-terminal", default=False, help="detatch or not to see container stdout")
+@click.option("--skip-cluster-management/--perform-cluster-management",
+              default=False, help="Skip cluster initialization/tear-down (only for kind-k8s deployments)")
 @click.argument('extra_args', nargs=-1)  # help: command: up <service1> <service2>
 @click.pass_context
-def start(ctx, stay_attached, extra_args):
+def start(ctx, stay_attached, skip_cluster_management, extra_args):
     ctx.obj = make_deploy_context(ctx)
     services_list = list(extra_args) or None
-    up_operation(ctx, services_list, stay_attached)
+    up_operation(ctx, services_list, stay_attached, skip_cluster_management)
 
 
+# TODO: remove legacy up command since it's an alias for stop
 @command.command()
 @click.option("--delete-volumes/--preserve-volumes", default=False, help="delete data volumes")
+@click.option("--skip-cluster-management/--perform-cluster-management",
+              default=False, help="Skip cluster initialization/tear-down (only for kind-k8s deployments)")
 @click.argument('extra_args', nargs=-1)  # help: command: down <service1> <service2>
 @click.pass_context
-def down(ctx, delete_volumes, extra_args):
+def down(ctx, delete_volumes, skip_cluster_management, extra_args):
     # Get the stack config file name
     # TODO: add cluster name and env file here
     ctx.obj = make_deploy_context(ctx)
-    down_operation(ctx, delete_volumes, extra_args)
+    down_operation(ctx, delete_volumes, extra_args, skip_cluster_management)
 
 
 # stop is the preferred alias for down
 @command.command()
 @click.option("--delete-volumes/--preserve-volumes", default=False, help="delete data volumes")
+@click.option("--skip-cluster-management/--perform-cluster-management",
+              default=False, help="Skip cluster initialization/tear-down (only for kind-k8s deployments)")
 @click.argument('extra_args', nargs=-1)  # help: command: down <service1> <service2>
 @click.pass_context
-def stop(ctx, delete_volumes, extra_args):
+def stop(ctx, delete_volumes, skip_cluster_management, extra_args):
     # TODO: add cluster name and env file here
     ctx.obj = make_deploy_context(ctx)
-    down_operation(ctx, delete_volumes, extra_args)
+    down_operation(ctx, delete_volumes, extra_args, skip_cluster_management)
 
 
 @command.command()
@@ -123,6 +133,7 @@ def push_images(ctx):
 @click.argument('extra_args', nargs=-1)  # help: command: port <service1> <service2>
 @click.pass_context
 def port(ctx, extra_args):
+    ctx.obj = make_deploy_context(ctx)
     port_operation(ctx, extra_args)
 
 
@@ -156,3 +167,14 @@ def status(ctx):
 def update(ctx):
     ctx.obj = make_deploy_context(ctx)
     update_operation(ctx)
+
+
+@command.command()
+@click.argument('job_name')
+@click.option('--helm-release', help='Helm release name (only for k8s helm chart deployments, defaults to chart name)')
+@click.pass_context
+def run_job(ctx, job_name, helm_release):
+    '''run a one-time job from the stack'''
+    from stack_orchestrator.deploy.deploy import run_job_operation
+    ctx.obj = make_deploy_context(ctx)
+    run_job_operation(ctx, job_name, helm_release)

stack_orchestrator/deploy/deployment_context.py

@@ -56,9 +56,8 @@ class DeploymentContext:
         self.stack.init_from_file(self.get_stack_file())
         deployment_file_path = self.get_deployment_file()
         if deployment_file_path.exists():
-            with deployment_file_path:
-                obj = get_yaml().load(open(deployment_file_path, "r"))
-                self.id = obj[constants.cluster_id_key]
+            obj = get_yaml().load(open(deployment_file_path, "r"))
+            self.id = obj[constants.cluster_id_key]
         # Handle the case of a legacy deployment with no file
         # Code below is intended to match the output from _make_default_cluster_name()
         # TODO: remove when we no longer need to support legacy deployments

stack_orchestrator/deploy/deployment_create.py

@@ -24,10 +24,10 @@ from secrets import token_hex
 import sys
 from stack_orchestrator import constants
 from stack_orchestrator.opts import opts
-from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
+from stack_orchestrator.util import (get_stack_path, get_parsed_deployment_spec, get_parsed_stack_config,
                                      global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
                                      get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file,
-                                     resolve_config_dir)
+                                     resolve_config_dir, get_job_list, get_job_file_path)
 from stack_orchestrator.deploy.spec import Spec
 from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand
 from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator
@@ -238,6 +238,11 @@ def _find_extra_config_dirs(parsed_pod_file, pod):
                         config_dir = host_path.split("/")[2]
                         if config_dir != pod:
                             config_dirs.add(config_dir)
+        for env_file in service_info.get("env_file", []):
+            if env_file.startswith("../config"):
+                config_dir = env_file.split("/")[2]
+                if config_dir != pod:
+                    config_dirs.add(config_dir)
     return config_dirs
 
 
@@ -438,23 +443,25 @@ def _check_volume_definitions(spec):
 @click.command()
 @click.option("--spec-file", required=True, help="Spec file to use to create this deployment")
 @click.option("--deployment-dir", help="Create deployment files in this directory")
+@click.option("--helm-chart", is_flag=True, default=False, help="Generate Helm chart instead of deploying (k8s only)")
 # TODO: Hack
 @click.option("--network-dir", help="Network configuration supplied in this directory")
 @click.option("--initial-peers", help="Initial set of persistent peers")
 @click.pass_context
-def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
+def create(ctx, spec_file, deployment_dir, helm_chart, network_dir, initial_peers):
     deployment_command_context = ctx.obj
-    return create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers)
+    return create_operation(deployment_command_context, spec_file, deployment_dir, helm_chart, network_dir, initial_peers)
 
 
 # The init command's implementation is in a separate function so that we can
 # call it from other commands, bypassing the click decoration stuff
-def create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers):
+def create_operation(deployment_command_context, spec_file, deployment_dir, helm_chart, network_dir, initial_peers):
     parsed_spec = Spec(os.path.abspath(spec_file), get_parsed_deployment_spec(spec_file))
     _check_volume_definitions(parsed_spec)
     stack_name = parsed_spec["stack"]
     deployment_type = parsed_spec[constants.deploy_to_key]
-    stack_file = get_stack_file_path(stack_name)
+
+    stack_file = get_stack_path(stack_name).joinpath(constants.stack_file_name)
     parsed_stack = get_parsed_stack_config(stack_name)
     if opts.o.debug:
         print(f"parsed spec: {parsed_spec}")
@@ -467,8 +474,18 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
     os.mkdir(deployment_dir_path)
     # Copy spec file and the stack file into the deployment dir
     copyfile(spec_file, deployment_dir_path.joinpath(constants.spec_file_name))
-    copyfile(stack_file, deployment_dir_path.joinpath(os.path.basename(stack_file)))
+    copyfile(stack_file, deployment_dir_path.joinpath(constants.stack_file_name))
+
+    # Create deployment.yml with cluster-id
     _create_deployment_file(deployment_dir_path)
+
+    # Branch to Helm chart generation flow if --helm-chart flag is set
+    if deployment_type == "k8s" and helm_chart:
+        from stack_orchestrator.deploy.k8s.helm.chart_generator import generate_helm_chart
+        generate_helm_chart(stack_name, spec_file, deployment_dir_path)
+        return  # Exit early for helm chart generation
+
+    # Existing deployment flow continues unchanged
     # Copy any config varibles from the spec file into an env file suitable for compose
     _write_config_file(spec_file, deployment_dir_path.joinpath(constants.config_file_name))
     # Copy any k8s config file into the deployment dir
@@ -509,6 +526,38 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
             os.mkdir(destination_script_dir)
             script_paths = get_pod_script_paths(parsed_stack, pod)
             _copy_files_to_directory(script_paths, destination_script_dir)
+        if parsed_spec.is_kubernetes_deployment():
+            for configmap in parsed_spec.get_configmaps():
+                source_config_dir = resolve_config_dir(stack_name, configmap)
+                if os.path.exists(source_config_dir):
+                    destination_config_dir = deployment_dir_path.joinpath("configmaps", configmap)
+                    copytree(source_config_dir, destination_config_dir, dirs_exist_ok=True)
+        else:
+            # TODO: We should probably only do this if the volume is marked :ro.
+            for volume_name, volume_path in parsed_spec.get_volumes().items():
+                source_config_dir = resolve_config_dir(stack_name, volume_name)
+                # Only copy if the source exists and is _not_ empty.
+                if os.path.exists(source_config_dir) and os.listdir(source_config_dir):
+                    destination_config_dir = deployment_dir_path.joinpath(volume_path)
+                    # Only copy if the destination exists and _is_ empty.
+                    if os.path.exists(destination_config_dir) and not os.listdir(destination_config_dir):
+                        copytree(source_config_dir, destination_config_dir, dirs_exist_ok=True)
+
+    # Copy the job files into the deployment dir (for Docker deployments)
+    jobs = get_job_list(parsed_stack)
+    if jobs and not parsed_spec.is_kubernetes_deployment():
+        destination_compose_jobs_dir = deployment_dir_path.joinpath("compose-jobs")
+        os.mkdir(destination_compose_jobs_dir)
+        for job in jobs:
+            job_file_path = get_job_file_path(stack_name, parsed_stack, job)
+            if job_file_path and job_file_path.exists():
+                parsed_job_file = yaml.load(open(job_file_path, "r"))
+                _fixup_pod_file(parsed_job_file, parsed_spec, destination_compose_dir)
+                with open(destination_compose_jobs_dir.joinpath("docker-compose-%s.yml" % job), "w") as output_file:
+                    yaml.dump(parsed_job_file, output_file)
+                if opts.o.debug:
+                    print(f"Copied job compose file: {job}")
+
     # Delegate to the stack's Python code
     # The deploy create command doesn't require a --stack argument so we need to insert the
     # stack member here.
@@ -530,15 +579,17 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
 @click.option("--chain-id", help="The new chain id")
 @click.option("--key-name", help="Name for new node key")
 @click.option("--gentx-files", help="List of comma-delimited gentx filenames from other nodes")
+@click.option("--gentx-addresses", type=str, help="List of comma-delimited validator addresses for other nodes")
 @click.option("--genesis-file", help="Genesis file for the network")
 @click.option("--initialize-network", is_flag=True, default=False, help="Initialize phase")
 @click.option("--join-network", is_flag=True, default=False, help="Join phase")
+@click.option("--connect-network", is_flag=True, default=False, help="Connect phase")
 @click.option("--create-network", is_flag=True, default=False, help="Create phase")
 @click.option("--network-dir", help="Directory for network files")
 @click.argument('extra_args', nargs=-1)
 @click.pass_context
-def setup(ctx, node_moniker, chain_id, key_name, gentx_files, genesis_file, initialize_network, join_network, create_network,
-          network_dir, extra_args):
-    parmeters = LaconicStackSetupCommand(chain_id, node_moniker, key_name, initialize_network, join_network, create_network,
-                                         gentx_files, genesis_file, network_dir)
+def setup(ctx, node_moniker, chain_id, key_name, gentx_files, gentx_addresses, genesis_file, initialize_network, join_network,
+          connect_network, create_network, network_dir, extra_args):
+    parmeters = LaconicStackSetupCommand(chain_id, node_moniker, key_name, initialize_network, join_network, connect_network,
+                                         create_network, gentx_files, gentx_addresses, genesis_file, network_dir)
     call_stack_deploy_setup(ctx.obj, parmeters, extra_args)

stack_orchestrator/deploy/images.py

@@ -29,16 +29,29 @@ def _image_needs_pushed(image: str):
     return image.endswith(":local")
 
 
+def _remote_tag_for_image(image: str, remote_repo_url: str):
+    # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
+    major_parts = image.split("/", 2)
+    image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
+    (image_name, image_version) = image_name_with_version.split(":")
+    if image_version == "local":
+        return f"{remote_repo_url}/{image_name}:deploy"
+    else:
+        return image
+
+
+# Note: do not add any calls this function
 def remote_image_exists(remote_repo_url: str, local_tag: str):
     docker = DockerClient()
     try:
-        remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
+        remote_tag = _remote_tag_for_image(local_tag, remote_repo_url)
         result = docker.manifest.inspect(remote_tag)
         return True if result else False
     except Exception:  # noqa: E722
         return False
 
 
+# Note: do not add any calls this function
 def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
     if not additional_tags:
         return
@@ -47,18 +60,20 @@ def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
         raise Exception(f"{local_tag} does not exist in {remote_repo_url}")
 
     docker = DockerClient()
-    remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
-    new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
+    remote_tag = _remote_tag_for_image(local_tag, remote_repo_url)
+    new_remote_tags = [_remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
     docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags)
 
 
-def remote_tag_for_image(image: str, remote_repo_url: str):
+def remote_tag_for_image_unique(image: str, remote_repo_url: str, deployment_id: str):
     # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
     major_parts = image.split("/", 2)
     image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
     (image_name, image_version) = image_name_with_version.split(":")
     if image_version == "local":
-        return f"{remote_repo_url}/{image_name}:deploy"
+        # Salt the tag with part of the deployment id to make it unique to this deployment
+        deployment_tag = deployment_id[-8:]
+        return f"{remote_repo_url}/{image_name}:deploy-{deployment_tag}"
     else:
         return image
 
@@ -73,14 +88,14 @@ def push_images_operation(command_context: DeployCommandContext, deployment_cont
     docker = DockerClient()
     for image in images:
         if _image_needs_pushed(image):
-            remote_tag = remote_tag_for_image(image, remote_repo_url)
+            remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id)
             if opts.o.verbose:
                 print(f"Tagging {image} to {remote_tag}")
             docker.image.tag(image, remote_tag)
     # Run docker push commands to upload
     for image in images:
         if _image_needs_pushed(image):
-            remote_tag = remote_tag_for_image(image, remote_repo_url)
+            remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id)
             if opts.o.verbose:
                 print(f"Pushing image {remote_tag}")
             docker.image.push(remote_tag)

stack_orchestrator/deploy/k8s/cluster_info.py

@@ -14,6 +14,7 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 import os
+import base64
 
 from kubernetes import client
 from typing import Any, List, Set
@@ -26,7 +27,7 @@ from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variable
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment
 from stack_orchestrator.deploy.deploy_types import DeployEnvVars
 from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits
-from stack_orchestrator.deploy.images import remote_tag_for_image
+from stack_orchestrator.deploy.images import remote_tag_for_image_unique
 
 DEFAULT_VOLUME_RESOURCES = Resources({
     "reservations": {"storage": "2Gi"}
@@ -78,28 +79,40 @@ class ClusterInfo:
         if (opts.o.debug):
             print(f"Env vars: {self.environment_variables.map}")
 
-    def get_nodeport(self):
+    def get_nodeports(self):
+        nodeports = []
         for pod_name in self.parsed_pod_yaml_map:
             pod = self.parsed_pod_yaml_map[pod_name]
             services = pod["services"]
             for service_name in services:
                 service_info = services[service_name]
                 if "ports" in service_info:
-                    port = int(service_info["ports"][0])
-                    if opts.o.debug:
-                        print(f"service port: {port}")
-        service = client.V1Service(
-            metadata=client.V1ObjectMeta(name=f"{self.app_name}-nodeport"),
-            spec=client.V1ServiceSpec(
-                type="NodePort",
-                ports=[client.V1ServicePort(
-                    port=port,
-                    target_port=port
-                )],
-                selector={"app": self.app_name}
-            )
-        )
-        return service
+                    for raw_port in [str(p) for p in service_info["ports"]]:
+                        if opts.o.debug:
+                            print(f"service port: {raw_port}")
+                        if ":" in raw_port:
+                            parts = raw_port.split(":")
+                            if len(parts) != 2:
+                                raise Exception(f"Invalid port definition: {raw_port}")
+                            node_port = int(parts[0])
+                            pod_port = int(parts[1])
+                        else:
+                            node_port = None
+                            pod_port = int(raw_port)
+                        service = client.V1Service(
+                            metadata=client.V1ObjectMeta(name=f"{self.app_name}-nodeport-{pod_port}"),
+                            spec=client.V1ServiceSpec(
+                                type="NodePort",
+                                ports=[client.V1ServicePort(
+                                    port=pod_port,
+                                    target_port=pod_port,
+                                    node_port=node_port
+                                )],
+                                selector={"app": self.app_name}
+                            )
+                        )
+                        nodeports.append(service)
+        return nodeports
 
     def get_ingress(self, use_tls=False, certificate=None, cluster_issuer="letsencrypt-prod"):
         # No ingress for a deployment that has no http-proxy defined, for now
@@ -248,12 +261,12 @@ class ClusterInfo:
             for f in os.listdir(cfg_map_path):
                 full_path = os.path.join(cfg_map_path, f)
                 if os.path.isfile(full_path):
-                    data[f] = open(full_path, 'rt').read()
+                    data[f] = base64.b64encode(open(full_path, 'rb').read()).decode('ASCII')
 
             spec = client.V1ConfigMap(
                 metadata=client.V1ObjectMeta(name=f"{self.app_name}-{cfg_map_name}",
                                              labels={"configmap-label": cfg_map_name}),
-                data=data
+                binary_data=data
             )
             result.append(spec)
         return result
@@ -326,8 +339,11 @@ class ClusterInfo:
                 if opts.o.debug:
                     print(f"Merged envs: {envs}")
                 # Re-write the image tag for remote deployment
-                image_to_use = remote_tag_for_image(
-                    image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
+                # Note self.app_name has the same value as deployment_id
+                image_to_use = remote_tag_for_image_unique(
+                    image,
+                    self.spec.get_image_registry(),
+                    self.app_name) if self.spec.get_image_registry() is not None else image
                 volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name)
                 container = client.V1Container(
                     name=container_name,
@@ -350,6 +366,8 @@ class ClusterInfo:
 
         annotations = None
         labels = {"app": self.app_name}
+        affinity = None
+        tolerations = None
 
         if self.spec.get_annotations():
             annotations = {}
@@ -362,17 +380,60 @@ class ClusterInfo:
                 for service_name in services:
                     labels[key.replace("{name}", service_name)] = value
 
+        if self.spec.get_node_affinities():
+            affinities = []
+            for rule in self.spec.get_node_affinities():
+                # TODO add some input validation here
+                label_name = rule['label']
+                label_value = rule['value']
+                affinities.append(client.V1NodeSelectorTerm(
+                            match_expressions=[client.V1NodeSelectorRequirement(
+                                key=label_name,
+                                operator="In",
+                                values=[label_value]
+                            )]
+                        )
+                    )
+            affinity = client.V1Affinity(
+                node_affinity=client.V1NodeAffinity(
+                    required_during_scheduling_ignored_during_execution=client.V1NodeSelector(
+                        node_selector_terms=affinities
+                    ))
+                )
+
+        if self.spec.get_node_tolerations():
+            tolerations = []
+            for toleration in self.spec.get_node_tolerations():
+                # TODO add some input validation here
+                toleration_key = toleration['key']
+                toleration_value = toleration['value']
+                tolerations.append(client.V1Toleration(
+                    effect="NoSchedule",
+                    key=toleration_key,
+                    operator="Equal",
+                    value=toleration_value
+                ))
+
         template = client.V1PodTemplateSpec(
             metadata=client.V1ObjectMeta(
                 annotations=annotations,
                 labels=labels
             ),
-            spec=client.V1PodSpec(containers=containers, image_pull_secrets=image_pull_secrets, volumes=volumes),
+            spec=client.V1PodSpec(
+                containers=containers,
+                image_pull_secrets=image_pull_secrets,
+                volumes=volumes,
+                affinity=affinity,
+                tolerations=tolerations
+                ),
         )
         spec = client.V1DeploymentSpec(
-            replicas=1, template=template, selector={
+            replicas=self.spec.get_replicas(),
+            template=template, selector={
                 "matchLabels":
-                {"app": self.app_name}})
+                {"app": self.app_name}
+            }
+        )
 
         deployment = client.V1Deployment(
             api_version="apps/v1",

stack_orchestrator/deploy/k8s/deploy_k8s.py

@@ -16,6 +16,7 @@ from datetime import datetime, timezone
 
 from pathlib import Path
 from kubernetes import client, config
+from typing import List
 
 from stack_orchestrator import constants
 from stack_orchestrator.deploy.deployer import Deployer, DeployerConfigGenerator
@@ -51,12 +52,14 @@ class K8sDeployer(Deployer):
     networking_api: client.NetworkingV1Api
     k8s_namespace: str = "default"
     kind_cluster_name: str
+    skip_cluster_management: bool
     cluster_info: ClusterInfo
     deployment_dir: Path
     deployment_context: DeploymentContext
 
     def __init__(self, type, deployment_context: DeploymentContext, compose_files, compose_project_name, compose_env_file) -> None:
         self.type = type
+        self.skip_cluster_management = False
         # TODO: workaround pending refactoring above to cope with being created with a null deployment_context
         if deployment_context is None:
             return
@@ -182,6 +185,7 @@ class K8sDeployer(Deployer):
         if len(host_parts) == 2:
             host_as_wild = f"*.{host_parts[1]}"
 
+        # TODO: resolve method deprecation below
         now = datetime.utcnow().replace(tzinfo=timezone.utc)
         fmt = "%Y-%m-%dT%H:%M:%S%z"
 
@@ -202,15 +206,16 @@ class K8sDeployer(Deployer):
                                 return cert
         return None
 
-    def up(self, detach, services):
+    def up(self, detach, skip_cluster_management, services):
+        self.skip_cluster_management = skip_cluster_management
         if not opts.o.dry_run:
-            if self.is_kind():
+            if self.is_kind() and not self.skip_cluster_management:
                 # Create the kind cluster
                 create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath(constants.kind_config_filename))
                 # Ensure the referenced containers are copied into kind
                 load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set)
             self.connect_api()
-            if self.is_kind():
+            if self.is_kind() and not self.skip_cluster_management:
                 # Now configure an ingress controller (not installed by default in kind)
                 install_ingress_for_kind()
                 # Wait for ingress to start (deployment provisioning will fail unless this is done)
@@ -246,8 +251,8 @@ class K8sDeployer(Deployer):
             if opts.o.debug:
                 print("No ingress configured")
 
-        nodeport: client.V1Service = self.cluster_info.get_nodeport()
-        if nodeport:
+        nodeports: List[client.V1Service] = self.cluster_info.get_nodeports()
+        for nodeport in nodeports:
             if opts.o.debug:
                 print(f"Sending this nodeport: {nodeport}")
             if not opts.o.dry_run:
@@ -259,7 +264,8 @@ class K8sDeployer(Deployer):
                     print("NodePort created:")
                     print(f"{nodeport_resp}")
 
-    def down(self, timeout, volumes):  # noqa: C901
+    def down(self, timeout, volumes, skip_cluster_management):  # noqa: C901
+        self.skip_cluster_management = skip_cluster_management
         self.connect_api()
         # Delete the k8s objects
 
@@ -342,10 +348,10 @@ class K8sDeployer(Deployer):
             if opts.o.debug:
                 print("No ingress to delete")
 
-        nodeport: client.V1Service = self.cluster_info.get_nodeport()
-        if nodeport:
+        nodeports: List[client.V1Service] = self.cluster_info.get_nodeports()
+        for nodeport in nodeports:
             if opts.o.debug:
-                print(f"Deleting this nodeport: {ingress}")
+                print(f"Deleting this nodeport: {nodeport}")
             try:
                 self.core_api.delete_namespaced_service(
                     namespace=self.k8s_namespace,
@@ -357,7 +363,7 @@ class K8sDeployer(Deployer):
             if opts.o.debug:
                 print("No nodeport to delete")
 
-        if self.is_kind():
+        if self.is_kind() and not self.skip_cluster_management:
             # Destroy the kind cluster
             destroy_cluster(self.kind_cluster_name)
 
@@ -504,6 +510,26 @@ class K8sDeployer(Deployer):
         # We need to figure out how to do this -- check why we're being called first
         pass
 
+    def run_job(self, job_name: str, helm_release: str = None):
+        if not opts.o.dry_run:
+            from stack_orchestrator.deploy.k8s.helm.job_runner import run_helm_job
+
+            # Check if this is a helm-based deployment
+            chart_dir = self.deployment_dir / "chart"
+            if not chart_dir.exists():
+                # TODO: Implement job support for compose-based K8s deployments
+                raise Exception(f"Job support is only available for helm-based deployments. Chart directory not found: {chart_dir}")
+
+            # Run the job using the helm job runner
+            run_helm_job(
+                chart_dir=chart_dir,
+                job_name=job_name,
+                release=helm_release,
+                namespace=self.k8s_namespace,
+                timeout=600,
+                verbose=opts.o.verbose
+            )
+
     def is_kind(self):
         return self.type == "k8s-kind"
 

stack_orchestrator/deploy/k8s/helm/__init__.py

@@ -0,0 +1,14 @@
+# Copyright © 2025 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.

stack_orchestrator/deploy/k8s/helm/chart_generator.py

@@ -0,0 +1,320 @@
+# Copyright © 2025 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from pathlib import Path
+
+from stack_orchestrator import constants
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import (
+    get_parsed_stack_config,
+    get_pod_list,
+    get_pod_file_path,
+    get_job_list,
+    get_job_file_path,
+    error_exit
+)
+from stack_orchestrator.deploy.k8s.helm.kompose_wrapper import (
+    check_kompose_available,
+    get_kompose_version,
+    convert_to_helm_chart
+)
+from stack_orchestrator.util import get_yaml
+
+
+def _wrap_job_templates_with_conditionals(chart_dir: Path, jobs: list) -> None:
+    """
+    Wrap job templates with conditional checks so they are not created by default.
+    Jobs will only be created when explicitly enabled via --set jobs.<name>.enabled=true
+    """
+    templates_dir = chart_dir / "templates"
+    if not templates_dir.exists():
+        return
+
+    for job_name in jobs:
+        # Find job template file (kompose generates <service-name>-job.yaml)
+        job_template_file = templates_dir / f"{job_name}-job.yaml"
+
+        if not job_template_file.exists():
+            if opts.o.debug:
+                print(f"Warning: Job template not found: {job_template_file}")
+            continue
+
+        # Read the template content
+        content = job_template_file.read_text()
+
+        # Wrap with conditional (default false)
+        # Use 'index' function to handle job names with dashes
+        # Provide default dict for .Values.jobs to handle case where it doesn't exist
+        condition = (
+            f"{{{{- if (index (.Values.jobs | default dict) "
+            f'"{job_name}" | default dict).enabled | default false }}}}'
+        )
+        wrapped_content = f"""{condition}
+{content}{{{{- end }}}}
+"""
+
+        # Write back
+        job_template_file.write_text(wrapped_content)
+
+        if opts.o.debug:
+            print(f"Wrapped job template with conditional: {job_template_file.name}")
+
+
+def _post_process_chart(chart_dir: Path, chart_name: str, jobs: list) -> None:
+    """
+    Post-process Kompose-generated chart to fix common issues.
+
+    Fixes:
+    1. Chart.yaml name, description and keywords
+    2. Add conditional wrappers to job templates (default: disabled)
+
+    TODO:
+    - Add defaultMode: 0755 to ConfigMap volumes containing scripts (.sh files)
+    """
+    yaml = get_yaml()
+
+    # Fix Chart.yaml
+    chart_yaml_path = chart_dir / "Chart.yaml"
+    if chart_yaml_path.exists():
+        chart_yaml = yaml.load(open(chart_yaml_path, "r"))
+
+        # Fix name
+        chart_yaml["name"] = chart_name
+
+        # Fix description
+        chart_yaml["description"] = f"Generated Helm chart for {chart_name} stack"
+
+        # Fix keywords
+        if "keywords" in chart_yaml and isinstance(chart_yaml["keywords"], list):
+            chart_yaml["keywords"] = [chart_name]
+
+        with open(chart_yaml_path, "w") as f:
+            yaml.dump(chart_yaml, f)
+
+    # Process job templates: wrap with conditionals (default disabled)
+    if jobs:
+        _wrap_job_templates_with_conditionals(chart_dir, jobs)
+
+
+def generate_helm_chart(stack_path: str, spec_file: str, deployment_dir_path: Path) -> None:
+    """
+    Generate a self-sufficient Helm chart from stack compose files using Kompose.
+
+    Args:
+        stack_path: Path to the stack directory
+        spec_file: Path to the deployment spec file
+        deployment_dir_path: Deployment directory path (already created with deployment.yml)
+
+    Output structure:
+        deployment-dir/
+        ├── deployment.yml  # Contains cluster-id
+        ├── spec.yml        # Reference
+        ├── stack.yml       # Reference
+        └── chart/          # Self-sufficient Helm chart
+            ├── Chart.yaml
+            ├── README.md
+            └── templates/
+                └── *.yaml
+
+    TODO: Enhancements:
+    - Convert Deployments to StatefulSets for stateful services (zenithd, postgres)
+    - Add _helpers.tpl with common label/selector functions
+    - Enhance Chart.yaml with proper metadata (version, description, etc.)
+    """
+
+    parsed_stack = get_parsed_stack_config(stack_path)
+    stack_name = parsed_stack.get("name", stack_path)
+
+    # 1. Check Kompose availability
+    if not check_kompose_available():
+        error_exit("kompose not found in PATH.\n")
+
+    # 2. Read cluster-id from deployment.yml
+    deployment_file = deployment_dir_path / constants.deployment_file_name
+    if not deployment_file.exists():
+        error_exit(f"Deployment file not found: {deployment_file}")
+
+    yaml = get_yaml()
+    deployment_config = yaml.load(open(deployment_file, "r"))
+    cluster_id = deployment_config.get(constants.cluster_id_key)
+    if not cluster_id:
+        error_exit(f"cluster-id not found in {deployment_file}")
+
+    # 3. Derive chart name from stack name + cluster-id suffix
+    # Sanitize stack name for use in chart name
+    sanitized_stack_name = stack_name.replace("_", "-").replace(" ", "-")
+
+    # Extract hex suffix from cluster-id (after the prefix)
+    # cluster-id format: "laconic-<hex>" -> extract the hex part
+    cluster_id_suffix = cluster_id.split("-", 1)[1] if "-" in cluster_id else cluster_id
+
+    # Combine to create human-readable + unique chart name
+    chart_name = f"{sanitized_stack_name}-{cluster_id_suffix}"
+
+    if opts.o.debug:
+        print(f"Cluster ID: {cluster_id}")
+        print(f"Chart name: {chart_name}")
+
+    # 4. Get compose files from stack (pods + jobs)
+    pods = get_pod_list(parsed_stack)
+    if not pods:
+        error_exit(f"No pods found in stack: {stack_path}")
+
+    jobs = get_job_list(parsed_stack)
+
+    if opts.o.debug:
+        print(f"Found {len(pods)} pod(s) in stack: {pods}")
+        if jobs:
+            print(f"Found {len(jobs)} job(s) in stack: {jobs}")
+
+    compose_files = []
+    for pod in pods:
+        pod_file = get_pod_file_path(stack_path, parsed_stack, pod)
+        if not pod_file.exists():
+            error_exit(f"Pod file not found: {pod_file}")
+        compose_files.append(pod_file)
+        if opts.o.debug:
+            print(f"Found compose file: {pod_file.name}")
+
+    # Add job compose files
+    job_files = []
+    for job in jobs:
+        job_file = get_job_file_path(stack_path, parsed_stack, job)
+        if not job_file.exists():
+            error_exit(f"Job file not found: {job_file}")
+        compose_files.append(job_file)
+        job_files.append(job_file)
+        if opts.o.debug:
+            print(f"Found job compose file: {job_file.name}")
+
+    try:
+        version = get_kompose_version()
+        print(f"Using kompose version: {version}")
+    except Exception as e:
+        error_exit(f"Failed to get kompose version: {e}")
+
+    # 5. Create chart directory and invoke Kompose
+    chart_dir = deployment_dir_path / "chart"
+
+    print(f"Converting {len(compose_files)} compose file(s) to Helm chart using Kompose...")
+
+    try:
+        output = convert_to_helm_chart(
+            compose_files=compose_files,
+            output_dir=chart_dir,
+            chart_name=chart_name
+        )
+        if opts.o.debug:
+            print(f"Kompose output:\n{output}")
+    except Exception as e:
+        error_exit(f"Helm chart generation failed: {e}")
+
+    # 6. Post-process generated chart
+    _post_process_chart(chart_dir, chart_name, jobs)
+
+    # 7. Generate README.md with basic installation instructions
+    readme_content = f"""# {chart_name} Helm Chart
+
+Generated by laconic-so from stack: `{stack_path}`
+
+## Prerequisites
+
+- Kubernetes cluster (v1.27+)
+- Helm (v3.12+)
+- kubectl configured to access your cluster
+
+## Installation
+
+```bash
+# Install the chart
+helm install {chart_name} {chart_dir}
+
+# Alternatively, install with your own release name
+# helm install <your-release-name> {chart_dir}
+
+# Check deployment status
+kubectl get pods
+```
+
+## Upgrade
+
+To apply changes made to chart, perform upgrade:
+
+```bash
+helm upgrade {chart_name} {chart_dir}
+```
+
+## Uninstallation
+
+```bash
+helm uninstall {chart_name}
+```
+
+## Configuration
+
+The chart was generated from Docker Compose files using Kompose.
+
+### Customization
+
+Edit the generated template files in `templates/` to customize:
+- Image repositories and tags
+- Resource limits (CPU, memory)
+- Persistent volume sizes
+- Replica counts
+"""
+
+    readme_path = chart_dir / "README.md"
+    readme_path.write_text(readme_content)
+
+    if opts.o.debug:
+        print(f"Generated README: {readme_path}")
+
+    # 7. Success message
+    print(f"\n{'=' * 60}")
+    print("✓ Helm chart generated successfully!")
+    print(f"{'=' * 60}")
+    print("\nChart details:")
+    print(f"  Name:     {chart_name}")
+    print(f"  Location: {chart_dir.absolute()}")
+    print(f"  Stack:    {stack_path}")
+
+    # Count generated files
+    template_files = list((chart_dir / "templates").glob("*.yaml")) if (chart_dir / "templates").exists() else []
+    print(f"  Files:    {len(template_files)} template(s) generated")
+
+    print("\nDeployment directory structure:")
+    print(f"  {deployment_dir_path}/")
+    print("  ├── deployment.yml  (cluster-id)")
+    print("  ├── spec.yml        (reference)")
+    print("  ├── stack.yml       (reference)")
+    print("  └── chart/          (self-sufficient Helm chart)")
+
+    print("\nNext steps:")
+    print("  1. Review the chart:")
+    print(f"     cd {chart_dir}")
+    print("     cat Chart.yaml")
+    print("")
+    print("  2. Review generated templates:")
+    print("     ls templates/")
+    print("")
+    print("  3. Install to Kubernetes:")
+    print(f"     helm install {chart_name} {chart_dir}")
+    print("")
+    print("     # Or use your own release name")
+    print(f"     helm install <your-release-name> {chart_dir}")
+    print("")
+    print("  4. Check deployment:")
+    print("     kubectl get pods")
+    print("")

stack_orchestrator/deploy/k8s/helm/job_runner.py

@@ -0,0 +1,149 @@
+# Copyright © 2025 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import subprocess
+import tempfile
+import os
+import json
+from pathlib import Path
+from stack_orchestrator.util import get_yaml
+
+
+def get_release_name_from_chart(chart_dir: Path) -> str:
+    """
+    Read the chart name from Chart.yaml to use as the release name.
+
+    Args:
+        chart_dir: Path to the Helm chart directory
+
+    Returns:
+        Chart name from Chart.yaml
+
+    Raises:
+        Exception if Chart.yaml not found or name is missing
+    """
+    chart_yaml_path = chart_dir / "Chart.yaml"
+    if not chart_yaml_path.exists():
+        raise Exception(f"Chart.yaml not found: {chart_yaml_path}")
+
+    yaml = get_yaml()
+    chart_yaml = yaml.load(open(chart_yaml_path, "r"))
+
+    if "name" not in chart_yaml:
+        raise Exception(f"Chart name not found in {chart_yaml_path}")
+
+    return chart_yaml["name"]
+
+
+def run_helm_job(
+    chart_dir: Path,
+    job_name: str,
+    release: str = None,
+    namespace: str = "default",
+    timeout: int = 600,
+    verbose: bool = False
+) -> None:
+    """
+    Run a one-time job from a Helm chart.
+
+    This function:
+    1. Uses provided release name, or reads it from Chart.yaml if not provided
+    2. Uses helm template to render the job manifest with the job enabled
+    3. Applies the job manifest to the cluster
+    4. Waits for the job to complete
+
+    Args:
+        chart_dir: Path to the Helm chart directory
+        job_name: Name of the job to run (without -job suffix)
+        release: Optional Helm release name (defaults to chart name from Chart.yaml)
+        namespace: Kubernetes namespace
+        timeout: Timeout in seconds for job completion (default: 600)
+        verbose: Enable verbose output
+
+    Raises:
+        Exception if the job fails or times out
+    """
+    if not chart_dir.exists():
+        raise Exception(f"Chart directory not found: {chart_dir}")
+
+    # Use provided release name, or get it from Chart.yaml
+    if release is None:
+        release = get_release_name_from_chart(chart_dir)
+        if verbose:
+            print(f"Using release name from Chart.yaml: {release}")
+    else:
+        if verbose:
+            print(f"Using provided release name: {release}")
+
+    job_template_file = f"templates/{job_name}-job.yaml"
+
+    if verbose:
+        print(f"Running job '{job_name}' from helm chart: {chart_dir}")
+
+    # Use helm template to render the job manifest
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as tmp_file:
+        try:
+            # Render job template with job enabled
+            # Use --set-json to properly handle job names with dashes
+            jobs_dict = {job_name: {"enabled": True}}
+            values_json = json.dumps(jobs_dict)
+            helm_cmd = [
+                "helm", "template", release, str(chart_dir),
+                "--show-only", job_template_file,
+                "--set-json", f"jobs={values_json}"
+            ]
+
+            if verbose:
+                print(f"Running: {' '.join(helm_cmd)}")
+
+            result = subprocess.run(helm_cmd, check=True, capture_output=True, text=True)
+            tmp_file.write(result.stdout)
+            tmp_file.flush()
+
+            if verbose:
+                print(f"Generated job manifest:\n{result.stdout}")
+
+            # Parse the manifest to get the actual job name
+            yaml = get_yaml()
+            manifest = yaml.load(result.stdout)
+            actual_job_name = manifest.get("metadata", {}).get("name", job_name)
+
+            # Apply the job manifest
+            kubectl_apply_cmd = ["kubectl", "apply", "-f", tmp_file.name, "-n", namespace]
+            subprocess.run(kubectl_apply_cmd, check=True, capture_output=True, text=True)
+
+            if verbose:
+                print(f"Job {actual_job_name} created, waiting for completion...")
+
+            # Wait for job completion
+            wait_cmd = [
+                "kubectl", "wait", "--for=condition=complete",
+                f"job/{actual_job_name}",
+                f"--timeout={timeout}s",
+                "-n", namespace
+            ]
+
+            subprocess.run(wait_cmd, check=True, capture_output=True, text=True)
+
+            if verbose:
+                print(f"Job {job_name} completed successfully")
+
+        except subprocess.CalledProcessError as e:
+            error_msg = e.stderr if e.stderr else str(e)
+            raise Exception(f"Job failed: {error_msg}")
+        finally:
+            # Clean up temp file
+            if os.path.exists(tmp_file.name):
+                os.unlink(tmp_file.name)

stack_orchestrator/deploy/k8s/helm/kompose_wrapper.py

@@ -0,0 +1,109 @@
+# Copyright © 2025 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import subprocess
+import shutil
+from pathlib import Path
+from typing import List
+
+
+def check_kompose_available() -> bool:
+    """Check if kompose binary is available in PATH."""
+    return shutil.which("kompose") is not None
+
+
+def get_kompose_version() -> str:
+    """
+    Get the installed kompose version.
+
+    Returns:
+        Version string (e.g., "1.34.0")
+
+    Raises:
+        Exception if kompose is not available
+    """
+    if not check_kompose_available():
+        raise Exception("kompose not found in PATH")
+
+    result = subprocess.run(
+        ["kompose", "version"],
+        capture_output=True,
+        text=True,
+        timeout=10
+    )
+
+    if result.returncode != 0:
+        raise Exception(f"Failed to get kompose version: {result.stderr}")
+
+    # Parse version from output like "1.34.0 (HEAD)"
+    # Output format: "1.34.0 (HEAD)" or just "1.34.0"
+    version_line = result.stdout.strip()
+    version = version_line.split()[0] if version_line else "unknown"
+
+    return version
+
+
+def convert_to_helm_chart(compose_files: List[Path], output_dir: Path, chart_name: str = None) -> str:
+    """
+    Invoke kompose to convert Docker Compose files to a Helm chart.
+
+    Args:
+        compose_files: List of paths to docker-compose.yml files
+        output_dir: Directory where the Helm chart will be generated
+        chart_name: Optional name for the chart (defaults to directory name)
+
+    Returns:
+        stdout from kompose command
+
+    Raises:
+        Exception if kompose conversion fails
+    """
+    if not check_kompose_available():
+        raise Exception(
+            "kompose not found in PATH. "
+            "Install from: https://kompose.io/installation/"
+        )
+
+    # Ensure output directory exists
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Build kompose command
+    cmd = ["kompose", "convert"]
+
+    # Add all compose files
+    for compose_file in compose_files:
+        if not compose_file.exists():
+            raise Exception(f"Compose file not found: {compose_file}")
+        cmd.extend(["-f", str(compose_file)])
+
+    # Add chart flag and output directory
+    cmd.extend(["--chart", "-o", str(output_dir)])
+
+    # Execute kompose
+    result = subprocess.run(
+        cmd,
+        capture_output=True,
+        text=True,
+        timeout=60
+    )
+
+    if result.returncode != 0:
+        raise Exception(
+            f"Kompose conversion failed:\n"
+            f"Command: {' '.join(cmd)}\n"
+            f"Error: {result.stderr}"
+        )
+
+    return result.stdout