env

Otherwise we sometimes see errors like:

```
cerc-webapp-deployer:   File "/root/.shiv/laconic-so_0f937aa98c2748ef9af8585d6f441dbc01546ace0d6660cbb159d1e5040aeddf/site-packages/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py", line 671, in command
cerc-webapp-deployer:     shutil.rmtree(tempdir)
cerc-webapp-deployer:   File "/usr/lib/python3.10/shutil.py", line 725, in rmtree
cerc-webapp-deployer:     _rmtree_safe_fd(fd, path, onerror)
cerc-webapp-deployer:   File "/usr/lib/python3.10/shutil.py", line 681, in _rmtree_safe_fd
cerc-webapp-deployer:     onerror(os.unlink, fullname, sys.exc_info())
cerc-webapp-deployer:   File "/usr/lib/python3.10/shutil.py", line 679, in _rmtree_safe_fd
cerc-webapp-deployer:     os.unlink(entry.name, dir_fd=topfd)
cerc-webapp-deployer: FileNotFoundError: [Errno 2] No such file or directory: 'S.gpg-agent.extra'
```

Reviewed-on: cerc-io/stack-orchestrator#941
Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com>
Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>

.gitea/workflows/fixturenet-eth-plugeth-arm-test.yml

@@ -1,61 +0,0 @@
-name: Fixturenet-Eth-Plugeth-Arm-Test
-
-on:
-  push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-arm-test'
-  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
-    - cron: '2 14 * * *'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
-
-jobs:
-  test:
-    name: "Run an Ethereum plugeth fixturenet test"
-    runs-on: ubuntu-latest-arm
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: "Run fixturenet-eth tests"
-        run: ./tests/fixturenet-eth-plugeth/run-test.sh
-      - name: Notify Vulcanize Slack on CI failure
-        if: ${{ always() && github.ref_name == 'main' }}
-        uses: ravsamhq/notify-slack-action@v2
-        with:
-          status: ${{ job.status }}
-          notify_when: 'failure'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
-      - name: Notify DeepStack Slack on CI failure
-        if: ${{ always() && github.ref_name == 'main' }}
-        uses: ravsamhq/notify-slack-action@v2
-        with:
-          status: ${{ job.status }}
-          notify_when: 'failure'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/fixturenet-eth-test.yml

@@ -1,63 +0,0 @@
-name: Fixturenet-Eth-Test
-
-on:
-  push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.gitea/workflows/triggers/fixturenet-eth-test'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
-
-jobs:
-  test:
-    name: "Run an Ethereum fixturenet test"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
-      - name: "Run fixturenet-eth tests"
-        run: ./tests/fixturenet-eth/run-test.sh
-      - name: Notify Vulcanize Slack on CI failure
-        if: ${{ always() && github.ref_name == 'main' }}
-        uses: ravsamhq/notify-slack-action@v2
-        with:
-          status: ${{ job.status }}
-          notify_when: 'failure'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
-      - name: Notify DeepStack Slack on CI failure
-        if: ${{ always() && github.ref_name == 'main' }}
-        uses: ravsamhq/notify-slack-action@v2
-        with:
-          status: ${{ job.status }}
-          notify_when: 'failure'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-deploy.yml

@@ -10,9 +10,6 @@ on:
     paths-ignore:
       - '.gitea/workflows/triggers/*'
 
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
 
 jobs:
   test:
@@ -41,10 +38,6 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run deploy tests"
         run: ./tests/deploy/run-deploy-test.sh
       - name: Notify Vulcanize Slack on CI failure

.gitea/workflows/test-k8s-deployment-control.yml

@@ -1,23 +1,22 @@
-name: Fixturenet-Eth-Plugeth-Test
+name: K8s Deployment Control Test
 
 on:
+  pull_request:
+    branches: '*'
   push:
     branches: '*'
     paths:
       - '!**'
-      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
+      - '.gitea/workflows/triggers/test-k8s-deployment-control'
+      - '.gitea/workflows/test-k8s-deployment-control.yml'
+      - 'tests/k8s-deployment-control/run-test.sh'
   schedule: # Note: coordinate with other tests to not overload runners at the same time of day
-    - cron: '2 14 * * *'
+    - cron: '3 30 * * *'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
 
 jobs:
   test:
-    name: "Run an Ethereum plugeth fixturenet test"
+    name: "Run deployment control suite on kind/k8s"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: "Clone project repository"
         uses: actions/checkout@v3
@@ -41,12 +40,17 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Run k8s deployment control test"
         run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          source /opt/bash-utils/cgroup-helper.sh
-          sleep 5
+          join_cgroup
-      - name: "Run fixturenet-eth tests"
+          ./tests/k8s-deployment-control/run-test.sh
-        run: ./tests/fixturenet-eth-plugeth/run-test.sh
       - name: Notify Vulcanize Slack on CI failure
         if: ${{ always() && github.ref_name == 'main' }}
         uses: ravsamhq/notify-slack-action@v2

.gitea/workflows/test-webapp.yml

@@ -10,10 +10,6 @@ on:
     paths-ignore:
       - '.gitea/workflows/triggers/*'
 
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
 jobs:
   test:
     name: "Run webapp test suite"
@@ -43,10 +39,6 @@ jobs:
         run: ./scripts/build_shiv_package.sh
       - name: "Install wget" # 20240109 - Only needed until the executors are updated.
         run: apt update && apt install -y wget
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run webapp tests"
         run: ./tests/webapp-test/run-webapp-test.sh
       - name: Notify Vulcanize Slack on CI failure

.gitea/workflows/test.yml

@@ -10,9 +10,6 @@ on:
     paths-ignore:
       - '.gitea/workflows/triggers/*'
 
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
 
 jobs:
   test:
@@ -41,10 +38,6 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run smoke tests"
         run: ./tests/smoke-test/run-smoke-test.sh
       - name: Notify Vulcanize Slack on CI failure

.gitea/workflows/triggers/fixturenet-eth-plugeth-arm-test

@@ -1,2 +0,0 @@
-Change this file to trigger running the fixturenet-eth-plugeth-arm-test CI job
-

.gitea/workflows/triggers/fixturenet-eth-plugeth-test

@@ -1,3 +0,0 @@
-Change this file to trigger running the fixturenet-eth-plugeth-test CI job
-trigger
-trigger

.gitea/workflows/triggers/fixturenet-eth-test

@@ -1,2 +0,0 @@
-Change this file to trigger running the fixturenet-eth-test CI job
-

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -4,3 +4,7 @@ Trigger
 Trigger
 Trigger
 Trigger
+Trigger
+Trigger
+Trigger
+Trigger

docs/cli.md

@@ -51,7 +51,7 @@ $ laconic-so build-npms --include <package-name>
 ```
 e.g.
 ```
-$ laconic-so build-npms --include laconic-sdk
+$ laconic-so build-npms --include registry-sdk
 ```
 Build the packages for a stack:
 ```

docs/gitea-with-laconicd-fixturenet.md

@@ -56,7 +56,7 @@ laconic-so --stack fixturenet-laconicd build-npms
 Navigate to the Gitea console and switch to the `cerc-io` user then find the `Packages` tab to confirm that these two npm packages have been published:
 
 - `@cerc-io/laconic-registry-cli`
-- `@cerc-io/laconic-sdk`
+- `@cerc-io/registry-sdk`
 
 ### Build and deploy fixturenet containers
 
@@ -74,7 +74,7 @@ laconic-so --stack fixturenet-laconicd deploy logs
 ### Test with the registry CLI
 
 ```bash
-laconic-so --stack fixturenet-laconicd deploy exec cli "laconic cns status"
+laconic-so --stack fixturenet-laconicd deploy exec cli "laconic registry status"
 ```
 
 Try additional CLI commands, documented [here](https://github.com/cerc-io/laconic-registry-cli#operations).

docs/k8s-deployment-enhancements.md

@@ -0,0 +1,27 @@
+# K8S Deployment Enhancements
+## Controlling pod placement
+The placement of pods created as part of a stack deployment can be controlled to either avoid certain nodes, or require certain nodes.
+### Pod/Node Affinity
+Node affinity rules applied to pods target node labels. The effect is that a pod can only be placed on a node having the specified label value. Note that other pods that do not have any node affinity rules can also be placed on those same nodes. Thus node affinity for a pod controls where that pod can be placed, but does not control where other pods are placed.
+
+Node affinity for stack pods is specified in the deployment's `spec.yml` file as follows:
+```
+node-affinities:
+  - label: nodetype
+    value: typeb
+```
+This example denotes that the stack's pods should only be placed on nodes that have the label `nodetype` with value `typeb`.
+### Node Taint Toleration
+K8s nodes can be given one or more "taints". These are special fields (distinct from labels) with a name (key) and optional value.
+When placing pods, the k8s scheduler will only assign a pod to a tainted node if the pod posesses a corresponding "toleration".
+This is metadata associated with the pod that specifies that the pod "tolerates" a given taint.
+Therefore taint toleration provides a mechanism by which only certain pods can be placed on specific nodes, and provides a complementary mechanism to node affinity.
+
+Taint toleration for stack pods is specified in the deployment's `spec.yml` file as follows:
+```
+node-tolerations:
+  - key: nodetype
+    value: typeb
+```
+This example denotes that the stack's pods will tolerate a taint: `nodetype=typeb`
+

docs/laconicd-with-console.md

@@ -1,9 +1,8 @@
 # Running a laconicd fixturenet with console
 
-The following tutorial explains the steps to run a laconicd fixturenet with CLI and web console that displays records in the registry. It is designed as an introduction to Stack Orchestrator and to showcase one component of the Laconic Stack. Prior to Stack Orchestrator, the following 4 repositories had to be cloned and setup manually:
+The following tutorial explains the steps to run a laconicd fixturenet with CLI and web console that displays records in the registry. It is designed as an introduction to Stack Orchestrator and to showcase one component of the Laconic Stack. Prior to Stack Orchestrator, the following repositories had to be cloned and setup manually:
 
 - https://git.vdb.to/cerc-io/laconicd
-- https://git.vdb.to/cerc-io/laconic-sdk
 - https://git.vdb.to/cerc-io/laconic-registry-cli
 - https://git.vdb.to/cerc-io/laconic-console
 
@@ -51,7 +50,7 @@ To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the
 1. Get the repositories
 
     ```
-    laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
+    laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd
     ```
 
 1. Build the containers:
@@ -76,6 +75,8 @@ To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the
 1. Create a deployment directory for the stack:
     ```
     laconic-so --stack fixturenet-laconic-loaded deploy init --output laconic-loaded.spec --map-ports-to-host any-same --config LACONIC_HOSTED_ENDPOINT=$BACKEND_ENDPOINT
+
+    # Update port mapping in the laconic-loaded.spec file to resolve port conflicts on host if any
     ```
     ```
     laconic-so --stack fixturenet-laconic-loaded deploy create --deployment-dir laconic-loaded-deployment --spec-file laconic-loaded.spec
@@ -95,52 +96,51 @@ To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the
     You'll see output from `laconicd` and the block height should be >1 to confirm it is running:
 
     ```
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:29PM INF indexed block exents height=12 module=txindex server=node
+    laconicd-1         | 6:12AM INF indexed block events height=16 module=txindex
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF Timed out dur=4976.960115 height=13 module=consensus round=0 server=node step=1
+    laconicd-1         | 6:12AM INF Timed out dur=2993.893332 height=17 module=consensus round=0 step=RoundStepNewHeight
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4","parts":{"hash":"22411A20B7F14CDA33244420FBDDAF24450C0628C7A06034FF22DAC3699DDCC8","total":1}},"height":13,"pol_round":-1,"round":0,"signature":"DEuqnaQmvyYbUwckttJmgKdpRu6eVm9i+9rQ1pIrV2PidkMNdWRZBLdmNghkIrUzGbW8Xd7UVJxtLRmwRASgBg==","timestamp":"2023-04-18T21:30:01.49450663Z"} server=node
+    laconicd-1         | 6:12AM INF received proposal module=consensus proposal="Proposal{17/0 (E15D03C180CE607AE8340A1325A0C134DFB4E1ADD992E173C701EBD362523267:1:DF138772FEF0, -1) 6A6F3B0A42B3 @ 2024-07-25T06:12:31.952967053Z}" proposer=86970D950BC9C16F3991A52D9C6DC55BA478A7C6
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received complete proposal block hash=D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4 height=13 module=consensus server=node
+    laconicd-1         | 6:12AM INF received complete proposal block hash=E15D03C180CE607AE8340A1325A0C134DFB4E1ADD992E173C701EBD362523267 height=17 module=consensus
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF finalizing commit of block hash={} height=13 module=consensus num_txs=0 root=1A8CA1AF139CCC80EC007C6321D8A63A46A793386EE2EDF9A5CA0AB2C90728B7 server=node
+    laconicd-1         | 6:12AM INF finalizing commit of block hash=E15D03C180CE607AE8340A1325A0C134DFB4E1ADD992E173C701EBD362523267 height=17 module=consensus num_txs=0 root=AF4941107DC718ED1425E77A3DC7F1154FB780B7A7DE20288DC43442203527E3
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF minted coins from module account amount=2059730459416582643aphoton from=mint module=x/bank
+    laconicd-1         | 6:12AM INF finalized block block_app_hash=26A665360BB1EE64E54F97F2A5AB7F621B33A86D9896574000C05DE63F43F788 height=17 module=state num_txs_res=0 num_val_updates=0
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF executed block height=13 module=state num_invalid_txs=0 num_valid_txs=0 server=node
+    laconicd-1         | 6:12AM INF executed block app_hash=26A665360BB1EE64E54F97F2A5AB7F621B33A86D9896574000C05DE63F43F788 height=17 module=state
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF commit synced commit=436F6D6D697449447B5B363520313037203630203232372039352038352032303820313334203231392032303520313433203130372031343920313431203139203139322038362031323720362031383520323533203137362031333820313735203135392031383620323334203135382031323120313431203230342037335D3A447D
+    laconicd-1         | 6:12AM INF committed state block_app_hash=AF4941107DC718ED1425E77A3DC7F1154FB780B7A7DE20288DC43442203527E3 height=17 module=state
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF committed state app_hash=416B3CE35F55D086DBCD8F6B958D13C0567F06B9FDB08AAF9FBAEA9E798DCC49 height=13 module=state num_txs=0 server=node
+    laconicd-1         | 6:12AM INF indexed block events height=17 module=txindex
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF indexed block exents height=13 module=txindex server=node
     ```
 
 4. Confirm operation of the registry CLI:
 
    ```
-   laconic-so deployment --dir laconic-loaded-deployment exec cli "laconic cns status"
+   laconic-so deployment --dir laconic-loaded-deployment exec cli "laconic registry status"
    ```
 
    ```
    {
      "version": "0.3.0",
      "node": {
-       "id": "4216af2ac9f68bda33a38803fc1b5c9559312c1d",
+       "id": "6e072894aa1f5d9535a1127a0d7a7f8e65100a2c",
        "network": "laconic_9000-1",
        "moniker": "localtestnet"
      },
      "sync": {
-       "latest_block_hash": "1BDF4CB9AE2390DA65BCF997C83133C18014FCDDCAE03708488F0B56FCEEA429",
+       "latestBlockHash": "260102C283D0411CFBA0270F7DC182650FFCA737A2F6F652A985F6065696F590",
-       "latest_block_height": "5",
+       "latestBlockHeight": "49",
-       "latest_block_time": "2023-08-09 16:00:30.386903172 +0000 UTC",
+       "latestBlockTime": "2024-07-25 06:14:05.626744215 +0000 UTC",
-       "catching_up": false
+       "catchingUp": false
      },
      "validator": {
-       "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
+       "address": "86970D950BC9C16F3991A52D9C6DC55BA478A7C6",
-       "voting_power": "1000000000000000"
+       "votingPower": "1000000000000000"
      },
      "validators": [
        {
-         "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
+         "address": "86970D950BC9C16F3991A52D9C6DC55BA478A7C6",
-         "voting_power": "1000000000000000",
+         "votingPower": "1000000000000000",
-         "proposer_priority": "0"
+         "proposerPriority": "0"
        }
      ],
-     "num_peers": "0",
+     "numPeers": "0",
      "peers": [],
-     "disk_usage": "292.0K"
+     "diskUsage": "688K"
    }
    ```
 
@@ -186,13 +186,13 @@ wns
 1. The following command will create a bond and publish a record:
 
 ```
-laconic-so --stack fixturenet-laconic-loaded deploy exec cli ./scripts/create-demo-records.sh
+laconic-so deployment --dir laconic-loaded-deployment exec cli ./scripts/create-demo-records.sh
 ```
 
 You'll get an output like:
 
 ```
-Balance is: 99998999999999998999600000
+Balance is: 9.9999e+25
 Created bond with id: dd88e8d6f9567b32b28e70552aea4419c5dd3307ebae85a284d1fe38904e301a
 Published demo-record-1.yml with id: bafyreierh3xnfivexlscdwubvczmddsnf46uytyfvrbdhkjzztvsz6ruly
 ```
@@ -223,5 +223,5 @@ record:
 - e.g,:
 
 ```
-laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns record list"
+laconic-so deployment --dir laconic-loaded-deployment exec cli "laconic registry record list"
 ```

requirements.txt

@@ -11,3 +11,5 @@ tomli==2.0.1
 validators==0.22.0
 kubernetes>=28.1.0
 humanfriendly>=10.0
+python-gnupg>=0.5.2
+requests>=2.3.2

setup.py

@@ -4,9 +4,11 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 with open("requirements.txt", "r", encoding="utf-8") as fh:
     requirements = fh.read()
+with open("stack_orchestrator/data/version.txt", "r", encoding="utf-8") as fh:
+    version = fh.readlines()[-1].strip(" \n")
 setup(
     name='laconic-stack-orchestrator',
-    version='1.0.12',
+    version=version,
     author='Cerc',
     author_email='info@cerc.io',
     license='GNU Affero General Public License',

stack_orchestrator/build/build_util.py

@@ -21,11 +21,6 @@ from stack_orchestrator.util import get_parsed_stack_config, warn_exit
 
 def get_containers_in_scope(stack: str):
 
-    # See: https://stackoverflow.com/a/20885799/1701505
-    from stack_orchestrator import data
-    with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
-        all_containers = container_list_file.read().splitlines()
-
     containers_in_scope = []
     if stack:
         stack_config = get_parsed_stack_config(stack)
@@ -33,7 +28,10 @@ def get_containers_in_scope(stack: str):
             warn_exit(f"stack {stack} does not define any containers")
         containers_in_scope = stack_config['containers']
     else:
-        containers_in_scope = all_containers
+        # See: https://stackoverflow.com/a/20885799/1701505
+        from stack_orchestrator import data
+        with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
+            containers_in_scope = container_list_file.read().splitlines()
 
     if opts.o.verbose:
         print(f'Containers: {containers_in_scope}')

stack_orchestrator/build/build_webapp.py

@@ -30,6 +30,12 @@ from stack_orchestrator.build import build_containers
 from stack_orchestrator.deploy.webapp.util import determine_base_container, TimedLogger
 from stack_orchestrator.build.build_types import BuildContext
 
+def create_env_file(env_vars, repo_root):
+    env_file_path = os.path.join(repo_root, '.env')
+    with open(env_file_path, 'w') as env_file:
+        for key, value in env_vars.items():
+            env_file.write(f"{key}={value}\n")
+    return env_file_path
 
 @click.command()
 @click.option('--base-container')
@@ -37,8 +43,9 @@ from stack_orchestrator.build.build_types import BuildContext
 @click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
 @click.option("--extra-build-args", help="Supply extra arguments to build")
 @click.option("--tag", help="Container tag (default: cerc/<app_name>:local)")
+@click.option("--env", help="Environment variables for webapp (format: KEY1=VALUE1,KEY2=VALUE2)", default="")
 @click.pass_context
-def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag):
+def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag, env):
     '''build the specified webapp container'''
     logger = TimedLogger()
 
@@ -88,9 +95,28 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     # Now build the target webapp.  We use the same build script, but with a different Dockerfile and work dir.
     container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
     container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo)
-    container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir,
+
-                                                                          base_container.replace("/", "-"),
+    # Check if Dockerfile exists in the repository
-                                                                          "Dockerfile.webapp")
+    repo_dockerfile = os.path.join(container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"], "Dockerfile")
+    default_dockerfile = os.path.join(container_build_dir,
+                                    base_container.replace("/", "-"),
+                                    "Dockerfile.webapp")
+
+    if os.path.isfile(repo_dockerfile):
+        env_vars = {}
+        if env:
+            for pair in env.split(','):
+                key, value = pair.split('=')
+                env_vars[key.strip()] = value.strip()
+        
+        container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = repo_dockerfile
+        
+        # Create .env file with environment variables
+        env_file_path = create_env_file(env_vars, container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"])
+        container_build_env["CERC_CONTAINER_BUILD_ENV_FILE"] = env_file_path
+    else:
+        container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = default_dockerfile
+
     if not tag:
         webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
         tag = f"cerc/{webapp_name}:local"

stack_orchestrator/constants.py

@@ -34,5 +34,8 @@ volumes_key = "volumes"
 security_key = "security"
 annotations_key = "annotations"
 labels_key = "labels"
+replicas_key = "replicas"
+node_affinities_key = "node-affinities"
+node_tolerations_key = "node-tolerations"
 kind_config_filename = "kind-config.yml"
 kube_config_filename = "kubeconfig.yml"

stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml

@@ -2,10 +2,11 @@ services:
   laconicd:
     restart: unless-stopped
     image: cerc/laconicd:local
-    command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
+    command: ["bash", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
     environment:
-      TEST_AUCTION_ENABLED: ${TEST_AUCTION_ENABLED}
+      TEST_AUCTION_ENABLED: ${TEST_AUCTION_ENABLED:-false}
-      TEST_REGISTRY_EXPIRY: ${TEST_REGISTRY_EXPIRY}
+      TEST_REGISTRY_EXPIRY: ${TEST_REGISTRY_EXPIRY:-false}
+      ONBOARDING_ENABLED: ${ONBOARDING_ENABLED:-false}
     volumes:
       # The cosmos-sdk node's database directory:
       - laconicd-data:/root/.laconicd
@@ -19,11 +20,9 @@ services:
       - "26657"
       - "26656"
       - "9473"
-      - "8545"
-      - "8546"
       - "9090"
-      - "9091"
       - "1317"
+
   cli:
     image: cerc/laconic-registry-cli:local
     volumes:

stack_orchestrator/data/compose/docker-compose-grafana.yml

@@ -6,12 +6,20 @@ services:
     restart: always
     environment:
       GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
+      CERC_GRAFANA_ALERTS_SUBGRAPH_IDS: ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
     volumes:
       - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
       - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards
+      - ../config/monitoring/update-grafana-alerts-config.sh:/update-grafana-alerts-config.sh
       - grafana_storage:/var/lib/grafana
+    user: root
+    entrypoint: ["bash", "-c"]
+    command: |
+      "/update-grafana-alerts-config.sh && /run.sh"
     ports:
       - "3000"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     healthcheck:
       test: ["CMD", "nc", "-vz", "localhost", "3000"]
       interval: 30s

stack_orchestrator/data/compose/docker-compose-graph-node.yml

@@ -22,6 +22,7 @@ services:
       GRAPH_ETHEREUM_JSON_RPC_TIMEOUT: ${GRAPH_ETHEREUM_JSON_RPC_TIMEOUT:-180}
       GRAPH_ETHEREUM_REQUEST_RETRIES: ${GRAPH_ETHEREUM_REQUEST_RETRIES:-10}
       GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE: ${GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE:-2000}
+      GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS: ${GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS:-1000}
     entrypoint: ["bash", "-c"]
     # Wait for ETH RPC endpoint to be up when running with fixturenet-lotus
     command: |
@@ -31,6 +32,7 @@ services:
       - "8001"
       - "8020"
       - "8030"
+      - "8040"
     healthcheck:
       test: ["CMD", "nc", "-vz", "localhost", "8020"]
       interval: 30s

stack_orchestrator/data/compose/docker-compose-laconic-explorer.yml

@@ -0,0 +1,10 @@
+services:
+  laconic-explorer:
+    restart: unless-stopped
+    image: cerc/ping-pub:local
+    environment:
+      - LACONIC_LACONICD_API_URL=${LACONIC_LACONICD_API_URL:-http://localhost:1317}
+      - LACONIC_LACONICD_RPC_URL=${LACONIC_LACONICD_RPC_URL:-http://localhost:26657}
+      - LACONIC_LACONICD_CHAIN_ID=${LACONIC_LACONICD_CHAIN_ID:-chain-id-not-set}
+    ports:
+      - "5173"

stack_orchestrator/data/compose/docker-compose-prom-server.yml

@@ -28,15 +28,37 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
-  chain-head-exporter:
+  ethereum-chain-head-exporter:
     image: cerc/watcher-ts:local
     restart: always
     working_dir: /app/packages/cli
     environment:
-      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT:-https://mainnet.infura.io/v3}
-      FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT}
       ETH_RPC_API_KEY: ${CERC_INFURA_KEY}
-      PORT: ${CERC_METRICS_PORT}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  filecoin-chain-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT:-https://api.node.glif.io/rpc/v1}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  graph-node-upstream-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${GRAPH_NODE_RPC_ENDPOINT}
     command: ["sh", "-c", "yarn export-metrics:chain-heads"]
     ports:
       - '5000'

stack_orchestrator/data/compose/docker-compose-snowballtools-base-backend.yml

@@ -1,13 +0,0 @@
-services:
-  snowballtools-base-backend:
-    image: cerc/snowballtools-base-backend:local
-    restart: always
-    volumes:
-      - data:/data
-      - config:/config:ro
-    ports:
-      - 8000
-
-volumes:
-  data:
-  config:

stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml

@@ -29,7 +29,7 @@ services:
     image: cerc/watcher-ajna:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
     command: ["bash", "./start-job-runner.sh"]
     volumes:
       - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
     ports:
       - "9000"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -55,16 +55,17 @@ services:
     image: cerc/watcher-ajna:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
     command: ["bash", "./start-server.sh"]
     volumes:
       - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
       - ../config/watcher-ajna/start-server.sh:/app/start-server.sh
+      - ajna_watcher_gql_logs_data:/app/gql-logs
     ports:
       - "3008"
       - "9001"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -74,3 +75,4 @@ services:
 
 volumes:
   ajna_watcher_db_data:
+  ajna_watcher_gql_logs_data:

stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml

@@ -32,8 +32,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CERC_HISTORICAL_BLOCK_RANGE: 500
       CONTRACT_ADDRESS: 0x223c067F8CF28ae173EE5CafEa60cA44C335fecB
       CONTRACT_NAME: Azimuth
@@ -47,7 +47,7 @@ services:
     ports:
       - "9000"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -66,18 +66,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/azimuth-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/azimuth-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/azimuth-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/azimuth-watcher/start-server.sh
+      - azimuth_watcher_gql_logs_data:/app/packages/azimuth-watcher/gql-logs
     ports:
       - "3001"
+      - "9001"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3001"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3001"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -94,8 +96,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0x325f68d32BdEe6Ed86E7235ff2480e2A433D6189
       CONTRACT_NAME: Censures
       STARTING_BLOCK: 6784954
@@ -108,7 +110,7 @@ services:
     ports:
       - "9002"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9002"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9002"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -127,18 +129,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/censures-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/censures-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/censures-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/censures-watcher/start-server.sh
+      - censures_watcher_gql_logs_data:/app/packages/censures-watcher/gql-logs
     ports:
       - "3002"
+      - "9003"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3002"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3002"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -155,8 +159,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0xe7e7f69b34D7d9Bd8d61Fb22C33b22708947971A
       CONTRACT_NAME: Claims
       STARTING_BLOCK: 6784941
@@ -169,7 +173,7 @@ services:
     ports:
       - "9004"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9004"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9004"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -188,18 +192,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/claims-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/claims-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/claims-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/claims-watcher/start-server.sh
+      - claims_watcher_gql_logs_data:/app/packages/claims-watcher/gql-logs
     ports:
       - "3003"
+      - "9005"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3003"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3003"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -216,8 +222,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0x8C241098C3D3498Fe1261421633FD57986D74AeA
       CONTRACT_NAME: ConditionalStarRelease
       STARTING_BLOCK: 6828004
@@ -230,7 +236,7 @@ services:
     ports:
       - "9006"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9006"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9006"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -249,18 +255,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/conditional-star-release-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/conditional-star-release-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/conditional-star-release-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/conditional-star-release-watcher/start-server.sh
+      - conditional_star_release_watcher_gql_logs_data:/app/packages/conditional-star-release-watcher/gql-logs
     ports:
       - "3004"
+      - "9007"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3004"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3004"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -277,8 +285,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0xf6b461fE1aD4bd2ce25B23Fe0aff2ac19B3dFA76
       CONTRACT_NAME: DelegatedSending
       STARTING_BLOCK: 6784956
@@ -291,7 +299,7 @@ services:
     ports:
       - "9008"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9008"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -310,18 +318,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/delegated-sending-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/delegated-sending-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/delegated-sending-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/delegated-sending-watcher/start-server.sh
+      - delegated_sending_watcher_gql_logs_data:/app/packages/delegated-sending-watcher/gql-logs
     ports:
       - "3005"
+      - "9009"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3005"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3005"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -338,8 +348,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0x33EeCbf908478C10614626A9D304bfe18B78DD73
       CONTRACT_NAME: Ecliptic
       STARTING_BLOCK: 13692129
@@ -352,7 +362,7 @@ services:
     ports:
       - "9010"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9010"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9010"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -371,18 +381,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/ecliptic-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/ecliptic-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/ecliptic-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/ecliptic-watcher/start-server.sh
+      - ecliptic_watcher_gql_logs_data:/app/packages/ecliptic-watcher/gql-logs
     ports:
       - "3006"
+      - "9011"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3006"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3006"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -399,8 +411,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0x86cd9cd0992F04231751E3761De45cEceA5d1801
       CONTRACT_NAME: LinearStarRelease
       STARTING_BLOCK: 6784943
@@ -413,7 +425,7 @@ services:
     ports:
       - "9012"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9012"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9012"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -432,18 +444,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/linear-star-release-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/linear-star-release-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/linear-star-release-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/linear-star-release-watcher/start-server.sh
+      - linear_star_release_watcher_gql_logs_data:/app/packages/linear-star-release-watcher/gql-logs
     ports:
       - "3007"
+      - "9013"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3007"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3007"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -460,8 +474,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0x7fEcaB617c868Bb5996d99D95200D2Fa708218e4
       CONTRACT_NAME: Polls
       STARTING_BLOCK: 6784912
@@ -474,7 +488,7 @@ services:
     ports:
       - "9014"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9014"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9014"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -493,18 +507,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/polls-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/polls-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/polls-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/polls-watcher/start-server.sh
+      - polls_watcher_gql_logs_data:/app/packages/polls-watcher/gql-logs
     ports:
       - "3008"
+      - "9015"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -542,7 +558,7 @@ services:
     ports:
       - "0.0.0.0:4000:4000"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "4000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "4000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -552,3 +568,11 @@ services:
 
 volumes:
   watcher_db_data:
+  azimuth_watcher_gql_logs_data:
+  censures_watcher_gql_logs_data:
+  claims_watcher_gql_logs_data:
+  conditional_star_release_watcher_gql_logs_data:
+  delegated_sending_watcher_gql_logs_data:
+  ecliptic_watcher_gql_logs_data:
+  linear_star_release_watcher_gql_logs_data:
+  polls_watcher_gql_logs_data:

stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml

@@ -29,7 +29,7 @@ services:
     image: cerc/watcher-merkl-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
     command: ["bash", "./start-job-runner.sh"]
     volumes:
       - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
     ports:
       - "9002:9000"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -55,16 +55,17 @@ services:
     image: cerc/watcher-merkl-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
     command: ["bash", "./start-server.sh"]
     volumes:
       - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
       - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh
+      - merkl_sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
     ports:
       - "127.0.0.1:3007:3008"
       - "9003:9001"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -74,3 +75,4 @@ services:
 
 volumes:
   merkl_sushiswap_v3_watcher_db_data:
+  merkl_sushiswap_v3_watcher_gql_logs_data:

stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml

@@ -29,7 +29,7 @@ services:
     image: cerc/watcher-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
     command: ["bash", "./start-job-runner.sh"]
     volumes:
       - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
     ports:
       - "9000:9000"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -55,16 +55,17 @@ services:
     image: cerc/watcher-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
     command: ["bash", "./start-server.sh"]
     volumes:
       - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
       - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh
+      - sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
     ports:
       - "127.0.0.1:3008:3008"
       - "9001:9001"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -74,3 +75,4 @@ services:
 
 volumes:
   sushiswap_v3_watcher_db_data:
+  sushiswap_v3_watcher_gql_logs_data:

stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh

@@ -8,68 +8,68 @@ KEY="mykey"
 CHAINID="laconic_9000-1"
 MONIKER="localtestnet"
 KEYRING="test"
-KEYALGO="eth_secp256k1"
+KEYALGO="secp256k1"
-LOGLEVEL="info"
+LOGLEVEL="${LOGLEVEL:-info}"
-# trace evm
+DENOM="alnt"
-TRACE="--trace"
+
-# TRACE=""
 
 if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then
   # validate dependencies are installed
-  command -v jq > /dev/null 2>&1 || { echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"; exit 1; }
+  command -v jq > /dev/null 2>&1 || {
+    echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"
+    exit 1
+  }
 
   # remove existing daemon and client
   rm -rf $HOME/.laconicd/*
-  rm -rf $HOME/.laconic/*
 
   if [ -n "`which make`" ]; then
     make install
   fi
 
-  laconicd config keyring-backend $KEYRING
+  laconicd config set client chain-id $CHAINID
-  laconicd config chain-id $CHAINID
+  laconicd config set client keyring-backend $KEYRING
 
   # if $KEY exists it should be deleted
   laconicd keys add $KEY --keyring-backend $KEYRING --algo $KEYALGO
 
   # Set moniker and chain-id for Ethermint (Moniker can be anything, chain-id must be an integer)
-  laconicd init $MONIKER --chain-id $CHAINID
+  laconicd init $MONIKER --chain-id $CHAINID --default-denom $DENOM
 
-  # Change parameter token denominations to aphoton
+  update_genesis() {
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["staking"]["params"]["bond_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    jq "$1" $HOME/.laconicd/config/genesis.json > $HOME/.laconicd/config/tmp_genesis.json &&
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["crisis"]["constant_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+      mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["gov"]["deposit_params"]["min_deposit"][0]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  }
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["mint"]["params"]["mint_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  # Custom modules
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commit_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveal_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_minimum_bid"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
 
   if [[ "$TEST_REGISTRY_EXPIRY" == "true" ]]; then
     echo "Setting timers for expiry tests."
 
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    update_genesis '.app_state["registry"]["params"]["record_rent_duration"]="60s"'
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    update_genesis '.app_state["registry"]["params"]["authority_grace_period"]="60s"'
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    update_genesis '.app_state["registry"]["params"]["authority_rent_duration"]="60s"'
   fi
 
   if [[ "$TEST_AUCTION_ENABLED" == "true" ]]; then
     echo "Enabling auction and setting timers."
 
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_enabled"]=true' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    update_genesis '.app_state["registry"]["params"]["authority_auction_enabled"]=true'
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    update_genesis '.app_state["registry"]["params"]["authority_rent_duration"]="60s"'
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="300s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    update_genesis '.app_state["registry"]["params"]["authority_grace_period"]="300s"'
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commits_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    update_genesis '.app_state["registry"]["params"]["authority_auction_commits_duration"]="60s"'
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveals_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    update_genesis '.app_state["registry"]["params"]["authority_auction_reveals_duration"]="60s"'
+  fi
+
+  if [[ "$ONBOARDING_ENABLED" == "true" ]]; then
+    echo "Enabling validator onboarding."
+
+    update_genesis '.app_state["onboarding"]["params"]["onboarding_enabled"]=true'
   fi
 
   # increase block time (?)
-  cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["time_iota_ms"]="1000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  update_genesis '.consensus["params"]["block"]["time_iota_ms"]="1000"'
 
   # Set gas limit in genesis
-  cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["max_gas"]="10000000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  update_genesis '.consensus["params"]["block"]["max_gas"]="10000000"'
 
   # disable produce empty block
   if [[ "$OSTYPE" == "darwin"* ]]; then
@@ -78,30 +78,6 @@ if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then
       sed -i 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml
   fi
 
-  if [[ $1 == "pending" ]]; then
-    if [[ "$OSTYPE" == "darwin"* ]]; then
-        sed -i '' 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml
-    else
-        sed -i 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml
-    fi
-  fi
-
   # Enable telemetry (prometheus metrics: http://localhost:1317/metrics?format=prometheus)
   if [[ "$OSTYPE" == "darwin"* ]]; then
     sed -i '' 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml
@@ -114,23 +90,27 @@ if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then
   fi
 
   # Allocate genesis accounts (cosmos formatted addresses)
-  laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING
+  # 10^30 alnt | 10^12 lnt
+  laconicd genesis add-genesis-account $KEY 1000000000000000000000000000000$DENOM --keyring-backend $KEYRING
 
   # Sign genesis transaction
-  laconicd gentx $KEY 1000000000000000000000aphoton --keyring-backend $KEYRING --chain-id $CHAINID
+  # 10^24 alnt | 10^6 lnt
+  laconicd genesis gentx $KEY 1000000000000000000000000$DENOM --keyring-backend $KEYRING --chain-id $CHAINID
 
   # Collect genesis tx
-  laconicd collect-gentxs
+  laconicd genesis collect-gentxs
 
   # Run this to ensure everything worked and that the genesis file is setup correctly
-  laconicd validate-genesis
+  laconicd genesis validate
-
-  if [[ $1 == "pending" ]]; then
-    echo "pending mode is on, please wait for the first block committed."
-  fi
 else
   echo "Using existing database at $HOME/.laconicd.  To replace, run '`basename $0` clean'"
 fi
 
 # Start the node (remove the --pruning=nothing flag if historical queries are not needed)
-laconicd start --pruning=nothing --evm.tracer=json $TRACE --log_level $LOGLEVEL --minimum-gas-prices=0.0001aphoton --json-rpc.api eth,txpool,personal,net,debug,web3,miner --api.enable --gql-server --gql-playground
+laconicd start \
+  --pruning=nothing \
+  --log_level $LOGLEVEL \
+  --minimum-gas-prices=1$DENOM \
+  --api.enable \
+  --rpc.laddr="tcp://0.0.0.0:26657" \
+  --gql-server --gql-playground

stack_orchestrator/data/config/fixturenet-laconicd/registry-cli-config-template.yml

@@ -1,9 +1,9 @@
 services:
-  cns:
+  registry:
-    restEndpoint: 'http://laconicd:1317'
+    rpcEndpoint: 'http://laconicd:26657'
     gqlEndpoint: 'http://laconicd:9473/api'
     userKey: REPLACE_WITH_MYKEY
     bondId:
     chainId: laconic_9000-1
     gas: 350000
-    fees: 200000aphoton
+    fees: 2000000alnt

stack_orchestrator/data/config/mainnet-laconicd/registry-cli-config-template.yml

@@ -1,9 +1,9 @@
 services:
-  cns:
+  registry:
-    restEndpoint: 'http://laconicd:1317'
+    rpcEndpoint: 'http://laconicd:26657'
     gqlEndpoint: 'http://laconicd:9473/api'
     userKey: REPLACE_WITH_MYKEY
     bondId:
     chainId: laconic_9000-1
     gas: 250000
-    fees: 200000aphoton
+    fees: 2000000alnt

stack_orchestrator/data/config/mainnet-laconicd/scripts/run-laconicd.sh

@@ -1,18 +1,15 @@
 #!/bin/sh
-if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
 #TODO: pass these in from the caller
-TRACE="--trace"
 LOGLEVEL="info"
 
 laconicd start \
     --pruning=nothing \
-    --evm.tracer=json $TRACE \
     --log_level $LOGLEVEL \
-    --minimum-gas-prices=0.0001aphoton \
+    --minimum-gas-prices=1alnt \
-    --json-rpc.api eth,txpool,personal,net,debug,web3,miner \
     --api.enable \
     --gql-server \
     --gql-playground

stack_orchestrator/data/config/monitoring/grafana/provisioning/datasources/graph-node-postgres.yml

@@ -0,0 +1,20 @@
+apiVersion: 1
+
+datasources:
+  - name: Graph Node Postgres
+    type: postgres
+    jsonData:
+      database: graph-node
+      sslmode: 'disable'
+      maxOpenConns: 100
+      maxIdleConns: 100
+      maxIdleConnsAuto: true
+      connMaxLifetime: 14400
+      postgresVersion: 1411 # 903=9.3, 1000=10, 1411=14.11
+      timescaledb: false
+    user: graph-node
+    # # Add URL for graph-node database
+    # url: graph-node-db:5432
+    # # Set password for graph-node database
+    # secureJsonData:
+    #   password: 'password'

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -45,7 +45,18 @@ scrape_configs:
     metrics_path: /metrics
     scheme: http
     static_configs:
-      - targets: ['chain-head-exporter:5000']
+      - targets: ['ethereum-chain-head-exporter:5000']
+        labels:
+          instance: 'external'
+          chain: 'ethereum'
+      - targets: ['filecoin-chain-head-exporter:5000']
+        labels:
+          instance: 'external'
+          chain: 'filecoin'
+      - targets: ['graph-node-upstream-head-exporter:5000']
+        labels:
+          instance: 'graph-node'
+          chain: 'filecoin'
 
   - job_name: 'postgres'
     scrape_interval: 30s
@@ -74,3 +85,11 @@ scrape_configs:
       # - targets: ['example-host:1317']
     params:
       format: ['prometheus']
+
+  - job_name: graph-node
+    metrics_path: /metrics
+    scrape_interval: 30s
+    scheme: http
+    static_configs:
+      # Add graph-node targets to be monitored below
+      # - targets: ['graph-node:8040']

stack_orchestrator/data/config/monitoring/subgraph-alert-rules.yml

@@ -0,0 +1,64 @@
+apiVersion: 1
+groups:
+  - orgId: 1
+    name: subgraph
+    folder: SubgraphAlerts
+    interval: 30s
+    rules:
+      - uid: b2a9144b-6104-46fc-92b5-352f4e643c4c
+        title: subgraph_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                  type: prometheus
+                  uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: ethereum_chain_head_number - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                  - evaluator:
+                      params:
+                          - 15
+                          - 0
+                      type: gt
+                    operator:
+                      type: and
+                    query:
+                      params: []
+                    reducer:
+                      params: []
+                      type: avg
+                    type: query
+              datasource:
+                  name: Expression
+                  type: __expr__
+                  uid: __expr__
+              expression: diff
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: threshold
+        noDataState: OK
+        execErrState: Alerting
+        for: 5m
+        annotations:
+          summary: Subgraph deployment {{ index $labels "deployment" }} is falling behind head by {{ index $values "diff" }}
+        labels: {}
+        isPaused: false

stack_orchestrator/data/config/monitoring/update-grafana-alerts-config.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+echo Using CERC_GRAFANA_ALERTS_SUBGRAPH_IDS ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
+
+# Replace subgraph ids in subgraph alerting config
+# Note: Requires the grafana container to be run with user root
+if [ -n "$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS" ]; then
+  sed -i "s/REPLACE_WITH_SUBGRAPH_IDS/$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS/g" /etc/grafana/provisioning/alerting/subgraph-alert-rules.yml
+fi

stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml

@@ -24,7 +24,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -100,7 +100,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -176,7 +176,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -252,7 +252,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -328,7 +328,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -404,7 +404,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -480,7 +480,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -556,7 +556,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -634,7 +634,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -710,7 +710,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -788,7 +788,85 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      
+      # Secured Finance
+      - uid: secured_finance_diff_external
+        title: secured_finance_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="secured-finance", instance="secured-finance", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true

stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh

@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-ajna/start-server.sh

@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml

@@ -2,7 +2,6 @@
   host = "0.0.0.0"
   port = 3008
   kind = "active"
-  gqlPath = "/"
 
   # Checkpointing state.
   checkpointing = true
@@ -22,23 +21,30 @@
   # Interval in number of blocks at which to clear entities cache.
   clearEntitiesCacheInterval = 1000
 
-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
   # Flag to specify whether RPC endpoint supports block hash as block tag parameter
   rpcSupportsBlockHashParam = false
 
-  # GQL cache settings
+  # Server GQL config
-  [server.gqlCache]
+  [server.gql]
-    enabled = true
+    path = "/"
 
-    # Max in-memory cache size (in bytes) (default 8 MB)
+    # Max block range for which to return events in eventsInRange GQL query.
-    # maxCacheSize
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000
 
-    # GQL cache-control max-age settings (in seconds)
+    # Log directory for GQL requests
-    maxAge = 15
+    logDir = "./gql-logs"
-    timeTravelMaxAge = 86400 # 1 day
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day
 
 [metrics]
   host = "0.0.0.0"
@@ -58,7 +64,7 @@
 
 [upstream]
   [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
 
     # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
     rpcClient = true
@@ -85,6 +91,9 @@
   # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
   blockDelayInMilliSecs = 30000
 
+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
   # Boolean to switch between modes of processing events when starting the server.
   # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
   # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -96,3 +105,6 @@
   # Max block range of historical processing after which it waits for completion of events processing
   # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
   historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3

stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh

@@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
-echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
-echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
+echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
 echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
 
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 # Replace env variables in template TOML file
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
-          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
+          s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
           s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
 
 # Write the modified content to a new file

stack_orchestrator/data/config/watcher-azimuth/start-server.sh

@@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
-echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
-echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
+echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
 echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
 
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 # Replace env variables in template TOML file
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
-          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
+          s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
           s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
 
 # Write the modified content to a new file

stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml

@@ -1,6 +1,7 @@
 [server]
   host = "0.0.0.0"
-  maxSimultaneousRequests = -1
+  [server.gql]
+    maxSimultaneousRequests = -1
 
 [metrics]
   host = "0.0.0.0"
@@ -13,8 +14,8 @@
 
 [upstream]
   [upstream.ethServer]
-    gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL"
+    gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT"
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
 
 [jobQueue]
   historicalLogsBlockRange = REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh

@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh

@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml

@@ -2,7 +2,6 @@
   host = "0.0.0.0"
   port = 3008
   kind = "active"
-  gqlPath = '/'
 
   # Checkpointing state.
   checkpointing = true
@@ -22,23 +21,30 @@
   # Interval in number of blocks at which to clear entities cache.
   clearEntitiesCacheInterval = 1000
 
-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
   # Flag to specify whether RPC endpoint supports block hash as block tag parameter
   rpcSupportsBlockHashParam = false
 
-  # GQL cache settings
+  # Server GQL config
-  [server.gqlCache]
+  [server.gql]
-    enabled = true
+    path = "/"
 
-    # Max in-memory cache size (in bytes) (default 8 MB)
+    # Max block range for which to return events in eventsInRange GQL query.
-    # maxCacheSize
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000
 
-    # GQL cache-control max-age settings (in seconds)
+    # Log directory for GQL requests
-    maxAge = 15
+    logDir = "./gql-logs"
-    timeTravelMaxAge = 86400 # 1 day
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day
 
 [metrics]
   host = "0.0.0.0"
@@ -58,7 +64,7 @@
 
 [upstream]
   [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
 
     # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
     rpcClient = true
@@ -69,7 +75,7 @@
     # Boolean flag to filter event logs by contracts
     filterLogsByAddresses = true
     # Boolean flag to filter event logs by topics
-    filterLogsByTopics = false
+    filterLogsByTopics = true
 
   [upstream.cache]
     name = "requests"
@@ -85,6 +91,9 @@
   # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
   blockDelayInMilliSecs = 30000
 
+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
   # Boolean to switch between modes of processing events when starting the server.
   # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
   # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -96,3 +105,6 @@
   # Max block range of historical processing after which it waits for completion of events processing
   # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
   historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3

stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh

@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh

@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml

@@ -2,7 +2,6 @@
   host = "0.0.0.0"
   port = 3008
   kind = "active"
-  gqlPath = "/"
 
   # Checkpointing state.
   checkpointing = true
@@ -22,23 +21,30 @@
   # Interval in number of blocks at which to clear entities cache.
   clearEntitiesCacheInterval = 1000
 
-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
   # Flag to specify whether RPC endpoint supports block hash as block tag parameter
   rpcSupportsBlockHashParam = false
 
-  # GQL cache settings
+  # Server GQL config
-  [server.gqlCache]
+  [server.gql]
-    enabled = true
+    path = "/"
 
-    # Max in-memory cache size (in bytes) (default 8 MB)
+    # Max block range for which to return events in eventsInRange GQL query.
-    # maxCacheSize
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000
 
-    # GQL cache-control max-age settings (in seconds)
+    # Log directory for GQL requests
-    maxAge = 15
+    logDir = "./gql-logs"
-    timeTravelMaxAge = 86400 # 1 day
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day
 
 [metrics]
   host = "0.0.0.0"
@@ -58,7 +64,7 @@
 
 [upstream]
   [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
 
     # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
     rpcClient = true
@@ -85,6 +91,9 @@
   # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
   blockDelayInMilliSecs = 30000
 
+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
   # Boolean to switch between modes of processing events when starting the server.
   # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
   # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -96,3 +105,6 @@
   # Max block range of historical processing after which it waits for completion of events processing
   # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
   historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3

stack_orchestrator/data/container-build/cerc-builder-js/Dockerfile

@@ -68,5 +68,5 @@ ENV PATH="${PATH}:/scripts"
 COPY entrypoint.sh .
 ENTRYPOINT ["./entrypoint.sh"]
 # Placeholder CMD : generally this will be overridden at run time like :
-# docker run -it -v /home/builder/cerc/laconic-sdk:/workspace cerc/builder-js sh -c 'cd /workspace && yarn && yarn build'
+# docker run -it -v /home/builder/cerc/registry-sdk:/workspace cerc/builder-js sh -c 'cd /workspace && yarn && yarn build'
 CMD node --version

stack_orchestrator/data/container-build/cerc-laconic-console-host/build.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Build cerc/laconic-registry-cli
+# Build cerc/laconic-console-host
 
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 

stack_orchestrator/data/container-build/cerc-laconic-registry-cli/create-demo-records.sh

@@ -5,16 +5,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
-registry_command="laconic cns"
+registry_command="laconic registry"
 demo_records_dir="scripts/demo-records"
 
 # Check we have funds
 funds_response=$(${registry_command} account get --address $(cat my-address.txt))
-funds_balance=$(echo ${funds_response} | jq -r .[0].balance[0].quantity)
+funds_balance=$(echo ${funds_response} | jq -r ".[0].balance[0].quantity")
 echo "Balance is: ${funds_balance}"
 
 # Create a bond
-bond_create_result=$(${registry_command} bond create --type aphoton --quantity 1000000000)
+bond_create_result=$(${registry_command} bond create --type alnt --quantity 1000000000)
 bond_id=$(echo ${bond_create_result} | jq -r .bondId)
 echo "Created bond with id: ${bond_id}"
 

stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-9.yml

@@ -7,9 +7,9 @@ record:
     env:
       ENV_VAR_A: A
       ENV_VAR_B: B
-  crn:
+  lrn:
-    - crn://foo.bar
+    - lrn://foo.bar
-    - crn://bar.baz
+    - lrn://bar.baz
   meta:
     foo: bar
   tags:

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -28,6 +28,8 @@ RUN \
     && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
     # Install pnpm
     && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
+    # Install bun
+    && su ${USERNAME} -c "umask 0002 && npm install -g bun@1.1.x" \
     && npm cache clean --force > /dev/null 2>&1
 
 # [Optional] Uncomment this section to install additional OS packages.

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh

@@ -14,6 +14,8 @@ if [ -z "$CERC_BUILD_TOOL" ]; then
     CERC_BUILD_TOOL=pnpm
   elif [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
+  elif [ -f "bun.lockb" ]; then
+    CERC_BUILD_TOOL=bun
   else
     CERC_BUILD_TOOL=npm
   fi

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -5,14 +5,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 
 CERC_MIN_NEXTVER=13.4.2
+CERC_DEFAULT_WEBPACK_VER="5.93.0"
 
 CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}"
+CERC_WEBPACK_VERSION="${CERC_WEBPACK_VERSION:-keep}"
+
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
   if [ -f "pnpm-lock.yaml" ]; then
     CERC_BUILD_TOOL=pnpm
   elif [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
+  elif [ -f "bun.lockb" ]; then
+    CERC_BUILD_TOOL=bun
   else
     CERC_BUILD_TOOL=npm
   fi
@@ -23,13 +28,21 @@ WORK_DIR="${1:-/app}"
 
 cd "${WORK_DIR}" || exit 1
 
+if [ -f "next.config.mjs" ]; then
+  NEXT_CONFIG_JS="next.config.mjs"
+  IMPORT_OR_REQUIRE="import"
+else
+  NEXT_CONFIG_JS="next.config.js"
+  IMPORT_OR_REQUIRE="require"
+fi
+
 # If this file doesn't exist at all, we'll get errors below.
-if [ ! -f "next.config.js" ]; then
+if [ ! -f "${NEXT_CONFIG_JS}" ]; then
-  touch next.config.js
+  touch ${NEXT_CONFIG_JS}
 fi
 
 if [ ! -f "next.config.dist" ]; then
-  cp next.config.js next.config.dist
+  cp $NEXT_CONFIG_JS next.config.dist
 fi
 
 which js-beautify >/dev/null
@@ -37,17 +50,34 @@ if [ $? -ne 0 ]; then
   npm i -g js-beautify
 fi
 
-js-beautify next.config.dist > next.config.js
+# js-beautify formats NEXTJS_CONFIG_FILE (ie next.config.js / next.config.mjs) so we can reliably transformable later
-echo "" >> next.config.js
+js-beautify next.config.dist > ${NEXT_CONFIG_JS}
+echo "" >> ${NEXT_CONFIG_JS}
 
-WEBPACK_REQ_LINE=$(grep -n "require([\'\"]webpack[\'\"])" next.config.js | cut -d':' -f1)
+if [ "${IMPORT_OR_REQUIRE}" == "require" ]; then
-if [ -z "$WEBPACK_REQ_LINE" ]; then
+  WEBPACK_REQ_LINE=$(grep -n "require([\'\"]webpack[\'\"])" ${NEXT_CONFIG_JS} | cut -d':' -f1)
-  cat > next.config.js.0 <<EOF
+  if [ -z "$WEBPACK_REQ_LINE" ]; then
+    cat > ${NEXT_CONFIG_JS}.0 <<EOF
     const webpack = require('webpack');
 EOF
+  fi
+else
+  WEBPACK_IMPORT_LINE=$(grep -n "^import .*[\'\"]webpack[\'\"];?$" ${NEXT_CONFIG_JS} | cut -d':' -f1)
+  if [ -z "$WEBPACK_IMPORT_LINE" ]; then
+    cat > ${NEXT_CONFIG_JS}.0 <<EOF
+    import webpack from 'webpack';
+EOF
+  fi
+  CREATE_REQUIRE_LINE=$(grep -n "require = createRequire" ${NEXT_CONFIG_JS} | cut -d':' -f1)
+  if [ -z "$CREATE_REQUIRE_LINE" ]; then
+    cat >> ${NEXT_CONFIG_JS}.0 <<EOF
+    import { createRequire } from "module";
+    const require = createRequire(import.meta.url);
+EOF
+  fi
 fi
 
-cat > next.config.js.1 <<EOF
+cat > ${NEXT_CONFIG_JS}.1 <<EOF
 let envMap;
 try {
   // .env-list.json provides us a list of identifiers which should be replaced at runtime.
@@ -55,7 +85,8 @@ try {
     a[v] = \`"CERC_RUNTIME_ENV_\${v.split(/\./).pop()}"\`;
     return a;
   }, {});
-} catch {
+} catch (e) {
+  console.error(e);
   // If .env-list.json cannot be loaded, we are probably running in dev mode, so use process.env instead.
   envMap = Object.keys(process.env).reduce((a, v) => {
     if (v.startsWith('CERC_')) {
@@ -64,40 +95,80 @@ try {
     return a;
   }, {});
 }
+console.log(envMap);
 EOF
 
-CONFIG_LINES=$(wc -l next.config.js | awk '{ print $1 }')
+grep 'withPWA' ${NEXT_CONFIG_JS} >/dev/null && HAS_WITHPWA=true || HAS_WITHPWA=false
-ENV_LINE=$(grep -n 'env:' next.config.js | cut -d':' -f1)
-WEBPACK_CONF_LINE=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d':' -f1)
-NEXT_SECTION_ADJUSTMENT=0
 
-if [ -n "$WEBPACK_CONF_LINE" ]; then
+if [ "$HAS_WITHPWA" == "true" ]; then
-  WEBPACK_CONF_VAR=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d',' -f1 | cut -d'(' -f2)
+  if [ "$IMPORT_OR_REQUIRE" == "import" ]; then
-  head -$(( ${WEBPACK_CONF_LINE} )) next.config.js > next.config.js.2
+    cat > ${NEXT_CONFIG_JS}.2 <<EOF
-  cat > next.config.js.3 <<EOF
+const __xPWA__ = (p) => {
-      $WEBPACK_CONF_VAR.plugins.push(new webpack.DefinePlugin(envMap));
+  const realPWA = withPWA(p);
-EOF
+  return (nextConfig) => {
-  NEXT_SECTION_LINE=$((WEBPACK_CONF_LINE))
+    const modConfig = {...nextConfig};
-elif [ -n "$ENV_LINE" ]; then
+
-  head -$(( ${ENV_LINE} - 1 )) next.config.js > next.config.js.2
+    modConfig.webpack = (config) => {
-  cat > next.config.js.3 <<EOF
-    webpack: (config) => {
       config.plugins.push(new webpack.DefinePlugin(envMap));
-      return config;
+      return nextConfig.webpack ? nextConfig.webpack(config) : config;
-    },
+    };
+
+    return realPWA(modConfig);
+  };
+};
 EOF
-  NEXT_SECTION_ADJUSTMENT=1
+  else
-  NEXT_SECTION_LINE=$ENV_LINE
+    cat > ${NEXT_CONFIG_JS}.3 <<EOF
+const __xPWA__ = (nextConfig) => {
+  const modConfig = {...nextConfig};
+
+  modConfig.webpack = (config) => {
+    config.plugins.push(new webpack.DefinePlugin(envMap));
+    return nextConfig.webpack ? nextConfig.webpack(config) : config;
+  };
+
+  return withPWA(modConfig);
+};
+EOF
+  fi
+
+  cat ${NEXT_CONFIG_JS} | js-beautify | sed 's/withPWA(/__xPWA__(/g' > ${NEXT_CONFIG_JS}.4
 else
-  echo "WARNING: Cannot find location to insert environment variable map in next.config.js" 1>&2
+    cat > ${NEXT_CONFIG_JS}.3 <<EOF
-  rm -f next.config.js.*
+  const __xCfg__ = (nextConfig) => {
-  NEXT_SECTION_LINE=0
+    const modConfig = {...nextConfig};
+
+    modConfig.webpack = (config) => {
+      config.plugins.push(new webpack.DefinePlugin(envMap));
+      return nextConfig.webpack ? nextConfig.webpack(config) : config;
+    };
+
+    return modConfig;
+  };
+EOF
+  if [ "$IMPORT_OR_REQUIRE" == "import" ]; then
+    cat ${NEXT_CONFIG_JS} | js-beautify | sed 's/export\s\+default\s\+/const __orig_cfg__ = /g' > ${NEXT_CONFIG_JS}.4
+    echo "export default __xCfg__(__orig_cfg__);" > ${NEXT_CONFIG_JS}.5
+  else
+    cat ${NEXT_CONFIG_JS} | js-beautify | sed 's/module.exports\s\+=\s\+/const __orig_cfg__ = /g' > ${NEXT_CONFIG_JS}.4
+    echo "module.exports = __xCfg__(__orig_cfg__);" > ${NEXT_CONFIG_JS}.5
+  fi
 fi
 
-tail -$(( ${CONFIG_LINES} - ${NEXT_SECTION_LINE} + ${NEXT_SECTION_ADJUSTMENT} )) next.config.js > next.config.js.5
 
-cat next.config.js.* | sed 's/^ *//g' | js-beautify | grep -v 'process\.\env\.' | js-beautify > next.config.js
+rm -f ${NEXT_CONFIG_JS}
-rm next.config.js.*
+for ((i=0; i <= 10; i++)); do
+  if [ -s "${NEXT_CONFIG_JS}.${i}" ]; then
+    if [ $i -le 2 ] ; then
+      cat ${NEXT_CONFIG_JS}.${i} >> ${NEXT_CONFIG_JS}
+    else
+      cat ${NEXT_CONFIG_JS}.${i} | sed 's/^ *//g' | js-beautify | grep -v 'process\.\env\.' | js-beautify >> ${NEXT_CONFIG_JS}
+    fi
+  fi
+done
+rm ${NEXT_CONFIG_JS}.*
+cat ${NEXT_CONFIG_JS} | js-beautify > ${NEXT_CONFIG_JS}.pretty
+mv ${NEXT_CONFIG_JS}.pretty ${NEXT_CONFIG_JS}
 
 "${SCRIPT_DIR}/find-env.sh" "$(pwd)" > .env-list.json
 
@@ -105,8 +176,6 @@ if [ ! -f "package.dist" ]; then
   cp package.json package.dist
 fi
 
-cat package.dist | jq '.scripts.cerc_compile = "next experimental-compile"' | jq '.scripts.cerc_generate = "next experimental-generate"' > package.json
-
 CUR_NEXT_VERSION="`jq -r '.dependencies.next' package.json`"
 
 if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VERSION" ]; then
@@ -115,10 +184,38 @@ if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VE
   mv package.json.$$ package.json
 fi
 
+CUR_WEBPACK_VERSION="`jq -r '.dependencies.webpack' package.json`"
+if [ -z "$CUR_WEBPACK_VERSION" ]; then
+  CUR_WEBPACK_VERSION="`jq -r '.devDependencies.webpack' package.json`"
+fi
+if [ "${CERC_WEBPACK_VERSION}" != "keep" ] || [ "${CUR_WEBPACK_VERSION}" == "null" ]; then
+  if [ -z "$CERC_WEBPACK_VERSION" ] || [ "$CERC_WEBPACK_VERSION" == "keep" ]; then
+    CERC_WEBPACK_VERSION="${CERC_DEFAULT_WEBPACK_VER}"
+  fi
+  echo "Webpack is required for env variable substitution.  Adding to webpack@$CERC_WEBPACK_VERSION to dependencies..." 1>&2
+  cat package.json | jq ".dependencies.webpack = \"$CERC_WEBPACK_VERSION\"" > package.json.$$
+  mv package.json.$$ package.json
+fi
+
 time $CERC_BUILD_TOOL install || exit 1
 
 CUR_NEXT_VERSION=`jq -r '.version' node_modules/next/package.json`
 
+# See https://github.com/vercel/next.js/discussions/46544
+semver -p -r ">=14.2.0" "$CUR_NEXT_VERSION"
+if [ $? -eq 0 ]; then
+  # For >= 14.2.0
+  CERC_NEXT_COMPILE_COMMAND="next build --experimental-build-mode compile"
+  CERC_NEXT_GENERATE_COMMAND="next build --experimental-build-mode generate"
+else
+  # For 13.4.2 to 14.1.x
+  CERC_NEXT_COMPILE_COMMAND="next experimental-compile"
+  CERC_NEXT_GENERATE_COMMAND="next experimental-generate"
+fi
+
+cat package.json | jq ".scripts.cerc_compile = \"$CERC_NEXT_COMPILE_COMMAND\"" | jq ".scripts.cerc_generate = \"$CERC_NEXT_GENERATE_COMMAND\"" > package.json.$$
+mv package.json.$$ package.json
+
 semver -p -r ">=$CERC_MIN_NEXTVER" $CUR_NEXT_VERSION
 if [ $? -ne 0 ]; then
   cat <<EOF

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh

@@ -20,9 +20,11 @@ for d in $(find . -maxdepth 1 -type d | grep -v '\./\.' | grep '/' | cut -d'/' -
   done
 done
 
-NEXT_CONF="next.config.js next.config.dist"
+NEXT_CONF="next.config.mjs next.config.js next.config.dist"
 for f in $NEXT_CONF; do
-    cat "$f" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o 'process.env.[A-Za-z0-9_]+' >> $TMPF
+    if [ -f "$f" ]; then
+        cat "$f" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o 'process.env.[A-Za-z0-9_]+' >> $TMPF
+    fi
 done
 
 cat $TMPF | sort -u | jq  --raw-input .  | jq --slurp .

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -5,7 +5,7 @@ fi
 
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-60}
+CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-120}
 tpid=""
 
 ctrl_c() {
@@ -20,6 +20,8 @@ if [ -z "$CERC_BUILD_TOOL" ]; then
     CERC_BUILD_TOOL=pnpm
   elif [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
+  elif [ -f "bun.lockb" ]; then
+    CERC_BUILD_TOOL=bun
   else
     CERC_BUILD_TOOL=npm
   fi

stack_orchestrator/data/container-build/cerc-ping-pub/Dockerfile

@@ -0,0 +1,10 @@
+FROM cerc/ping-pub-base:local
+
+COPY ./scripts/update-explorer-config.sh /scripts
+COPY ./scripts/start-serving-explorer.sh /scripts
+COPY ./config/laconic-chaindata-template.json /config/chains/laconic-chaindata-template.json
+
+EXPOSE 5173
+
+WORKDIR /app
+CMD ["/scripts/start-serving-explorer.sh"]

stack_orchestrator/data/container-build/cerc-ping-pub/Dockerfile.base

@@ -0,0 +1,8 @@
+FROM cerc/webapp-base:local
+
+WORKDIR /app
+
+COPY . .
+
+RUN yarn
+

stack_orchestrator/data/container-build/cerc-ping-pub/build.sh

@@ -1,5 +1,12 @@
 #!/usr/bin/env bash
 # Build the ping pub image
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-docker build -t cerc/ping-pub:local ${build_command_args} -f $CERC_REPO_BASE_DIR/explorer/Dockerfile $CERC_REPO_BASE_DIR/explorer
+# Two-stage build is to allow us to pick up both the upstream repo's files, and local files here for config
+docker build -t cerc/ping-pub-base:local ${build_command_args} -f $SCRIPT_DIR/Dockerfile.base $CERC_REPO_BASE_DIR/cosmos-explorer
+if [[ $? -ne 0 ]]; then
+    echo "FATAL: Base container build failed, exiting"
+    exit 1
+fi
+docker build -t cerc/ping-pub:local ${build_command_args} -f $SCRIPT_DIR/Dockerfile $SCRIPT_DIR

stack_orchestrator/data/container-build/cerc-ping-pub/config/laconic-chaindata-template.json

@@ -0,0 +1,22 @@
+{
+    "chain_name": "LACONIC_LACONICD_CHAIN_ID",
+    "registry_name": "LACONIC_LACONICD_CHAIN_ID",
+    "api": [
+        {"provider": "LX-one-tree-one-seven", "address": "LACONIC_LACONICD_API_URL"}
+    ],
+    "rpc": [
+        {"provider": "LX-tendermint-rpc", "address": "LACONIC_LACONICD_RPC_URL"}
+    ],
+    "sdk_version": "0.50.3",
+    "coin_type": "118",
+    "min_tx_fee": "800",
+    "addr_prefix": "laconic",
+    "logo": "/logos/cosmos.svg",
+    "assets": [{
+        "base": "alnt",
+        "symbol": "LNT",
+        "exponent": "18",
+        "coingecko_id": "cosmos",
+        "logo": "/logos/cosmos.svg"
+    }]
+}

stack_orchestrator/data/container-build/cerc-ping-pub/scripts/start-serving-explorer.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+${SCRIPT_DIR}/update-explorer-config.sh
+
+echo "Starting serving explorer"
+# Force cache re-build because vite is dumb and can't be restarted otherwise
+yarn serve --host --force

stack_orchestrator/data/container-build/cerc-ping-pub/scripts/update-explorer-config.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# Verify that we have the config variables we need
+if [[ -z ${LACONIC_LACONICD_API_URL} ]]; then
+    echo "Error: LACONIC_LACONICD_API_URL not defined"
+    exit 1
+fi
+if [[ -z ${LACONIC_LACONICD_RPC_URL} ]]; then
+    echo "Error: LACONIC_LACONICD_RPC_URL not defined"
+    exit 1
+fi
+if [[ -z ${LACONIC_LACONICD_CHAIN_ID} ]]; then
+    echo "Error: LACONIC_LACONICD_CHAIN_ID not defined"
+    exit 1
+fi
+
+# Ping-pub explorer has endlessly confusing behavior where it
+# infers the directory from which to load chain configuration files
+# by the presence or absense of the substring "testnet" in the host name
+# (browser side -- the host name of the host in the address bar of the browser)
+# Accordingly we configure our network in both directories in order to 
+# subvert this lunacy.
+explorer_mainnet_config_dir=/app/chains/mainnet
+explorer_testnet_config_dir=/app/chains/testnet
+config_template_file=/config/chains/laconic-chaindata-template.json
+chain_config_name=laconic.json
+mainnet_config_file=${explorer_mainnet_config_dir}/${chain_config_name}
+testnet_config_file=${explorer_testnet_config_dir}/${chain_config_name}
+
+# Delete the stock config files
+rm -f ${explorer_testnet_config_dir}/*
+rm -f ${explorer_mainnet_config_dir}/*
+
+# Copy in our template file
+cp ${config_template_file} ${mainnet_config_file}
+
+# Update the file with the config variables
+sed -i "s#LACONIC_LACONICD_API_URL#${LACONIC_LACONICD_API_URL}#g" ${mainnet_config_file}
+sed -i "s#LACONIC_LACONICD_RPC_URL#${LACONIC_LACONICD_RPC_URL}#g" ${mainnet_config_file}
+sed -i "s#LACONIC_LACONICD_CHAIN_ID#${LACONIC_LACONICD_CHAIN_ID}#g" ${mainnet_config_file}
+
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  echo "Updated chaindata file:"
+  cat ${mainnet_config_file}
+fi
+
+# Copy over to the testnet directory
+cp ${mainnet_config_file} ${testnet_config_file}

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/Dockerfile

@@ -1,6 +0,0 @@
-FROM cerc/snowballtools-base-backend-base:local
-
-WORKDIR /app/packages/backend
-COPY run.sh .
-
-ENTRYPOINT ["./run.sh"]

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/Dockerfile-base

@@ -1,26 +0,0 @@
-FROM ubuntu:22.04 as builder
-
-RUN apt update && \
-      apt install -y --no-install-recommends --no-install-suggests \
-      ca-certificates curl gnupg
-
-# Node
-ARG NODE_MAJOR=20
-RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
-      echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
-      apt update && apt install -y nodejs
-
-# npm setup
-RUN npm config set @cerc-io:registry https://git.vdb.to/api/packages/cerc-io/npm/ && npm install -g yarn
-
-COPY . /app/
-WORKDIR /app/
-
-RUN find . -name 'node_modules' | xargs -n1 rm -rf
-RUN yarn && yarn build --ignore frontend
-
-FROM cerc/webapp-base:local
-
-COPY --from=builder /app /app
-
-WORKDIR /app/packages/backend

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/build.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-# Build cerc/webapp-deployer-backend
-
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-
-# See: https://stackoverflow.com/a/246128/1701505
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-docker build -t cerc/snowballtools-base-backend-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile-base ${CERC_REPO_BASE_DIR}/snowballtools-base
-docker build -t cerc/snowballtools-base-backend:local ${build_command_args} ${SCRIPT_DIR} 

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/run.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-
-
-LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
-if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
-  if [ -f "/config/laconic-hosted-config.yml" ]; then
-    LACONIC_HOSTED_CONFIG_FILE="/config/laconic-hosted-config.yml"
-  elif [ -f "/config/config.yml" ]; then
-    LACONIC_HOSTED_CONFIG_FILE="/config/config.yml"
-  fi
-fi
-
-if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
-  /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "`pwd`/dist"
-fi
-
-/scripts/apply-runtime-env.sh "`pwd`/dist"
-
-yarn start

stack_orchestrator/data/container-build/cerc-watcher-ajna/Dockerfile

@@ -6,5 +6,10 @@ WORKDIR /app
 
 COPY . .
 
+# Get the latest Git commit hash and set in package.json
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+  jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
+  mv tmp.json package.json
+
 RUN echo "Installing dependencies and building ajna-watcher-ts" && \
   yarn && yarn build

stack_orchestrator/data/container-build/cerc-watcher-azimuth/Dockerfile

@@ -1,11 +1,20 @@
 FROM node:18.16.0-alpine3.16
 
-RUN apk --update --no-cache add git python3 alpine-sdk
+RUN apk --update --no-cache add git python3 alpine-sdk jq
 
 WORKDIR /app
 
 COPY . .
 
+# Get the latest Git commit hash and set it in package.json of all watchers
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+    find . -name 'package.json' -exec sh -c ' \
+    for packageFile; do \
+        jq --arg commitHash "$0" ".commitHash = \$commitHash" "$packageFile" > "$packageFile.tmp" && \
+        mv "$packageFile.tmp" "$packageFile"; \
+    done \
+    ' "$COMMIT_HASH" {} \;
+
 RUN echo "Building azimuth-watcher-ts" && \
     yarn && yarn build
 

stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile

@@ -6,5 +6,10 @@ WORKDIR /app
 
 COPY . .
 
+# Get the latest Git commit hash and set in package.json
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+  jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
+  mv tmp.json package.json
+
 RUN echo "Installing dependencies and building merkl-sushiswap-v3-watcher-ts" && \
   yarn && yarn build

stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/Dockerfile

@@ -6,5 +6,10 @@ WORKDIR /app
 
 COPY . .
 
+# Get the latest Git commit hash and set in package.json
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+  jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
+  mv tmp.json package.json
+
 RUN echo "Installing dependencies and building sushiswap-v3-watcher-ts" && \
   yarn && yarn build

stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile

@@ -28,11 +28,13 @@ RUN \
     && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
     # Install pnpm
     && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
+    # Install bun
+    && su ${USERNAME} -c "umask 0002 && npm install -g bun@1.1.x" \
     && npm cache clean --force > /dev/null 2>&1
 
 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base
+    && apt-get -y install --no-install-recommends jq gettext-base git
 
 # [Optional] Uncomment if you want to install an additional version of node using nvm
 # ARG EXTRA_NODE_VERSION=10

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/apply-runtime-env.sh

@@ -4,6 +4,8 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
+# TODO: document what this script does
+
 WORK_DIR="${1:-./}"
 
 cd "${WORK_DIR}" || exit 1

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh

@@ -27,6 +27,8 @@ elif [ -f "${WORK_DIR}/package.json" ]; then
       CERC_BUILD_TOOL=pnpm
     elif [ -f "yarn.lock" ]; then
       CERC_BUILD_TOOL=yarn
+    elif [ -f "bun.lockb" ]; then
+      CERC_BUILD_TOOL=bun
     else
       CERC_BUILD_TOOL=npm
     fi

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh

@@ -9,20 +9,26 @@ CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}"
 CERC_SINGLE_PAGE_APP="${CERC_SINGLE_PAGE_APP}"
 
 if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then 
-  if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ] && [ -d "${CERC_WEBAPP_FILES_DIR}/static" ]; then
+  # If there is only one HTML file, assume an SPA.
+  if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ]; then
     CERC_SINGLE_PAGE_APP=true
   else
     CERC_SINGLE_PAGE_APP=false
   fi
 fi
 
-if [ "true" == "$CERC_ENABLE_CORS" ]; then
+# ${var,,} is a lower-case comparison
+if [ "true" == "${CERC_ENABLE_CORS,,}" ]; then
   CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --cors"
 fi
 
-if [ "true" == "$CERC_SINGLE_PAGE_APP" ]; then
+# ${var,,} is a lower-case comparison
+if [ "true" == "${CERC_SINGLE_PAGE_APP,,}" ]; then
+  echo "Serving content as single-page app.  If this is wrong, set 'CERC_SINGLE_PAGE_APP=false'"
   # Create a catchall redirect back to /
   CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --proxy http://localhost:${CERC_LISTEN_PORT}?"
+else
+  echo "Serving content normally.  If this is a single-page app, set 'CERC_SINGLE_PAGE_APP=true'"
 fi
 
 LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}

stack_orchestrator/data/npm-package-list.txt

@@ -1,4 +1,4 @@
-laconic-sdk
+registry-sdk
 laconic-registry-cli
 laconic-console
 debug

stack_orchestrator/data/repository-list.txt

@@ -7,7 +7,7 @@ github.com/cerc-io/ipld-eth-db-validator
 github.com/cerc-io/ipld-eth-beacon-indexer
 github.com/cerc-io/ipld-eth-beacon-db
 github.com/cerc-io/laconicd
-github.com/cerc-io/laconic-sdk
+github.com/cerc-io/registry-sdk
 github.com/cerc-io/laconic-registry-cli
 github.com/cerc-io/laconic-console
 github.com/cerc-io/mobymask-watcher-ts

stack_orchestrator/data/stacks/ajna/README.md

@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v
 
 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
 ```
 
 ### Start the deployment

stack_orchestrator/data/stacks/ajna/stack.yml

@@ -2,7 +2,7 @@ version: "1.0"
 name: ajna
 description: "Ajna watcher stack"
 repos:
-  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.1
+  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.13
 containers:
   - cerc/watcher-ajna
 pods:

stack_orchestrator/data/stacks/azimuth/README.md

@@ -4,7 +4,7 @@ Instructions to setup and deploy Azimuth Watcher stack
 
 ## Setup
 
-Prerequisite: `ipld-eth-server` RPC and GQL endpoints
+Prerequisite: External RPC endpoints
 
 Clone required repositories:
 
@@ -44,34 +44,42 @@ network:
      - 0.0.0.0:9000:9000
     azimuth-watcher-server:
      - 0.0.0.0:3001:3001
+     - 0.0.0.0:9001:9001
     censures-watcher-job-runner:
      - 0.0.0.0:9002:9002
     censures-watcher-server:
      - 0.0.0.0:3002:3002
+     - 0.0.0.0:9003:9003
     claims-watcher-job-runner:
      - 0.0.0.0:9004:9004
     claims-watcher-server:
      - 0.0.0.0:3003:3003
+     - 0.0.0.0:9005:9005
     conditional-star-release-watcher-job-runner:
      - 0.0.0.0:9006:9006
     conditional-star-release-watcher-server:
      - 0.0.0.0:3004:3004
+     - 0.0.0.0:9007:9007
     delegated-sending-watcher-job-runner:
      - 0.0.0.0:9008:9008
     delegated-sending-watcher-server:
      - 0.0.0.0:3005:3005
+     - 0.0.0.0:9009:9009
     ecliptic-watcher-job-runner:
      - 0.0.0.0:9010:9010
     ecliptic-watcher-server:
      - 0.0.0.0:3006:3006
+     - 0.0.0.0:9011:9011
     linear-star-release-watcher-job-runner:
      - 0.0.0.0:9012:9012
     linear-star-release-watcher-server:
      - 0.0.0.0:3007:3007
+     - 0.0.0.0:9013:9013
     polls-watcher-job-runner:
      - 0.0.0.0:9014:9014
     polls-watcher-server:
      - 0.0.0.0:3008:3008
+     - 0.0.0.0:9015:9015
     gateway-server:
      - 0.0.0.0:4000:4000
 ...
@@ -94,7 +102,7 @@ Inside the deployment directory, open the file `config.env` and add variable to
 
   ```bash
   # External RPC endpoints
-  CERC_IPLD_ETH_RPC=
+  CERC_ETH_RPC_ENDPOINTS=https://example-rpc-endpoint-1,https://example-rpc-endpoint-2
   ```
 
 * NOTE: If RPC endpoint is on the host machine, use `host.docker.internal` as the hostname to access the host port, or use the `ip a` command to find the IP address of the `docker0` interface (this will usually be something like `172.17.0.1` or `172.18.0.1`)
@@ -120,4 +128,7 @@ To stop all azimuth services and also delete data:
 
 ```bash
 laconic-so deployment --dir azimuth-deployment stop --delete-volumes
+
+# Remove deployment directory (deployment will have to be recreated for a re-run)
+rm -r azimuth-deployment
 ```

stack_orchestrator/data/stacks/azimuth/stack.yml

@@ -1,7 +1,7 @@
 version: "1.0"
 name: azimuth
 repos:
-  - github.com/cerc-io/azimuth-watcher-ts@v0.1.3
+  - github.com/cerc-io/azimuth-watcher-ts@0.1.6
 containers:
   - cerc/watcher-azimuth
 pods:

stack_orchestrator/data/stacks/build-support/README.md

@@ -58,5 +58,5 @@ Now npm packages can be built:
 Ensure that `CERC_NPM_AUTH_TOKEN` is set with the token printed above when the package-registry stack was deployed (the actual token value will be different than shown in this example):
 ```
 $ export CERC_NPM_AUTH_TOKEN=84fe66a73698bf11edbdccd0a338236b7d1d5c45
-$ laconic-so build-npms --include laconic-sdk,laconic-registry-cli
+$ laconic-so build-npms --include registry-sdk,laconic-registry-cli
 ```

stack_orchestrator/data/stacks/fixturenet-laconic-loaded/README.md

@@ -49,7 +49,7 @@ $ laconic-so --stack fixturenet-laconic-loaded deploy logs
 ```
 ## 6. Test with the Registry CLI
 ```
-$ laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns status"
+$ laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic registry status"
 ```
 ## 7. View the laconic console
 Get the URL for the console web app with this command (the port number will be different for each deployment):
@@ -64,5 +64,6 @@ $ laconic-so --stack fixturenet-laconic-loaded deploy exec cli ./scripts/create-
 Balance is: 99998999999999998999600000
 Created bond with id: dd88e8d6f9567b32b28e70552aea4419c5dd3307ebae85a284d1fe38904e301a
 Published demo-record-1.yml with id: bafyreierh3xnfivexlscdwubvczmddsnf46uytyfvrbdhkjzztvsz6ruly
+...
 ```
-The published record should be visible in the console.
+The published records should be visible in the console.

stack_orchestrator/data/stacks/fixturenet-laconic-loaded/stack.yml

@@ -7,11 +7,11 @@ repos:
   - github.com/lirewine/crypto
   - github.com/lirewine/gem
   - github.com/lirewine/sdk
-  - git.vdb.to/cerc-io/laconic-sdk
+  - git.vdb.to/cerc-io/registry-sdk
   - git.vdb.to/cerc-io/laconic-registry-cli
   - git.vdb.to/cerc-io/laconic-console
 npms:
-  - laconic-sdk
+  - registry-sdk
   - laconic-registry-cli
   - debug
   - crypto
@@ -30,4 +30,3 @@ config:
   cli:
     key: laconicd.mykey
     address: laconicd.myaddress
-

stack_orchestrator/data/stacks/fixturenet-laconicd/README.md

@@ -44,5 +44,5 @@ $ laconic-so --stack fixturenet-laconicd deploy logs
 ```
 ## 6. Test with the Registry CLI
 ```
-$ laconic-so --stack fixturenet-laconicd deploy exec cli "laconic cns status"
+$ laconic-so --stack fixturenet-laconicd deploy exec cli "laconic registry status"
 ```

stack_orchestrator/data/stacks/fixturenet-laconicd/stack.yml

@@ -3,10 +3,10 @@ name: fixturenet-laconicd
 description: "A laconicd fixturenet"
 repos:
   - git.vdb.to/cerc-io/laconicd
-  - git.vdb.to/cerc-io/laconic-sdk
+  - git.vdb.to/cerc-io/registry-sdk
   - git.vdb.to/cerc-io/laconic-registry-cli
 npms:
-  - laconic-sdk
+  - registry-sdk
   - laconic-registry-cli
 containers:
   - cerc/laconicd

stack_orchestrator/data/stacks/graph-node/README.md

@@ -43,16 +43,18 @@ customized by editing the "spec" file generated by `laconic-so deploy init`.
 ```
 $ cat graph-node-spec.yml
 stack: graph-node
-ports:
+network:
-  graph-node:
+  ports:
-    - '8000:8000'
+    graph-node:
-    - '8001'
+      - '8000:8000'
-    - '8020:8020'
+      - '8001'
-    - '8030'
+      - '8020:8020'
-  ipfs:
+      - '8030'
-    - '8080'
+      - '8040'
-    - '4001'
+    ipfs:
-    - '5001:5001'
+      - '8080'
+      - '4001'
+      - '5001:5001'
 ...
 ```
 
@@ -64,7 +66,7 @@ laconic-so --stack graph-node deploy create --spec-file graph-node-spec.yml --de
 
 ## Start the stack
 
-Create an env file with the following values to be set before starting the stack:
+Update `config.env` file inside deployment directory with the following values before starting the stack:
 
 ```bash
 # Set ETH RPC endpoint the graph node will use
@@ -88,10 +90,13 @@ export GRAPH_ETHEREUM_REQUEST_RETRIES=
 # Maximum number of blocks to scan for triggers in each request (default: 2000)
 export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=
 
+# Maximum number of concurrent requests made against Ethereum for requesting transaction receipts during block ingestion (default: 1000)
+export GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS=
+
 # Ref: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
 ```
 
-Example env file:
+Example `config.env` file:
 
 ```bash
 export ETH_RPC_HOST=filecoin.chainup.net
@@ -104,12 +109,6 @@ export GRAPH_ETHEREUM_REQUEST_RETRIES=5
 export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=50
 ```
 
-Set the environment variables:
-
-```bash
-source <PATH_TO_ENV_FILE>
-```
-
 Deploy the stack:
 
 ```bash

stack_orchestrator/data/stacks/graph-node/stack.yml

@@ -3,7 +3,9 @@ name: graph-node
 description: "Stack for running graph-node"
 repos:
   - github.com/graphprotocol/graph-node
+  - github.com/cerc-io/watcher-ts@v0.2.92
 containers:
   - cerc/graph-node
+  - cerc/watcher-ts
 pods:
   - graph-node

stack_orchestrator/data/stacks/mainnet-laconic/deploy/commands.py

@@ -14,14 +14,14 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from stack_orchestrator.util import get_yaml
-from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
+from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand
+from stack_orchestrator.deploy.deployment_context import DeploymentContext
 from stack_orchestrator.deploy.stack_state import State
 from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command
-from stack_orchestrator.command_types import CommandOptions
+from stack_orchestrator.opts import opts
 from enum import Enum
 from pathlib import Path
 from shutil import copyfile, copytree
-import json
 import os
 import sys
 import tomli
@@ -33,8 +33,9 @@ default_spec_file_content = ""
 class SetupPhase(Enum):
     INITIALIZE = 1
     JOIN = 2
-    CREATE = 3
+    CONNECT = 3
-    ILLEGAL = 3
+    CREATE = 4
+    ILLEGAL = 5
 
 
 def _client_toml_path(network_dir: Path):
@@ -61,39 +62,23 @@ def _get_node_moniker_from_config(network_dir: Path):
     return moniker
 
 
-def _get_node_key_from_gentx(options: CommandOptions, gentx_file_name: str):
-    gentx_file_path = Path(gentx_file_name)
-    if gentx_file_path.exists():
-        with open(Path(gentx_file_name), "rb") as f:
-            parsed_json = json.load(f)
-            return parsed_json['body']['messages'][0]['delegator_address']
-    else:
-        print(f"Error: gentx file: {gentx_file_name} does not exist")
-        sys.exit(1)
-
-
 def _comma_delimited_to_list(list_str: str):
     return list_str.split(",") if list_str else []
 
 
-def _get_node_keys_from_gentx_files(options: CommandOptions, gentx_file_list: str):
+def _get_node_keys_from_gentx_files(gentx_address_list: str):
-    node_keys = []
+    gentx_addresses = _comma_delimited_to_list(gentx_address_list)
-    gentx_files = _comma_delimited_to_list(gentx_file_list)
+    return gentx_addresses
-    for gentx_file in gentx_files:
-        node_key = _get_node_key_from_gentx(options, gentx_file)
-        if node_key:
-            node_keys.append(node_key)
-    return node_keys
 
 
-def _copy_gentx_files(options: CommandOptions, network_dir: Path, gentx_file_list: str):
+def _copy_gentx_files(network_dir: Path, gentx_file_list: str):
     gentx_files = _comma_delimited_to_list(gentx_file_list)
     for gentx_file in gentx_files:
         gentx_file_path = Path(gentx_file)
         copyfile(gentx_file_path, os.path.join(network_dir, "config", "gentx", os.path.basename(gentx_file_path)))
 
 
-def _remove_persistent_peers(options: CommandOptions, network_dir: Path):
+def _remove_persistent_peers(network_dir: Path):
     config_file_path = _config_toml_path(network_dir)
     if not config_file_path.exists():
         print("Error: config.toml not found")
@@ -107,20 +92,74 @@ def _remove_persistent_peers(options: CommandOptions, network_dir: Path):
         output_file.write(config_file_content)
 
 
-def _insert_persistent_peers(options: CommandOptions, config_dir: Path, new_persistent_peers: str):
+def _insert_persistent_peers(config_dir: Path, new_persistent_peers: str):
     config_file_path = config_dir.joinpath("config.toml")
     if not config_file_path.exists():
         print("Error: config.toml not found")
         sys.exit(1)
     with open(config_file_path, "r") as input_file:
         config_file_content = input_file.read()
-        persistent_peers_pattern = '^persistent_peers = ""'
+        persistent_peers_pattern = r'^persistent_peers = ""'
         replace_with = f"persistent_peers = \"{new_persistent_peers}\""
         config_file_content = re.sub(persistent_peers_pattern, replace_with, config_file_content, flags=re.MULTILINE)
     with open(config_file_path, "w") as output_file:
         output_file.write(config_file_content)
 
 
+def _enable_cors(config_dir: Path):
+    config_file_path = config_dir.joinpath("config.toml")
+    if not config_file_path.exists():
+        print("Error: config.toml not found")
+        sys.exit(1)
+    with open(config_file_path, "r") as input_file:
+        config_file_content = input_file.read()
+        cors_pattern = r'^cors_allowed_origins = \[]'
+        replace_with = 'cors_allowed_origins = ["*"]'
+        config_file_content = re.sub(cors_pattern, replace_with, config_file_content, flags=re.MULTILINE)
+    with open(config_file_path, "w") as output_file:
+        output_file.write(config_file_content)
+    app_file_path = config_dir.joinpath("app.toml")
+    if not app_file_path.exists():
+        print("Error: app.toml not found")
+        sys.exit(1)
+    with open(app_file_path, "r") as input_file:
+        app_file_content = input_file.read()
+        cors_pattern = r'^enabled-unsafe-cors = false'
+        replace_with = "enabled-unsafe-cors = true"
+        app_file_content = re.sub(cors_pattern, replace_with, app_file_content, flags=re.MULTILINE)
+    with open(app_file_path, "w") as output_file:
+        output_file.write(app_file_content)
+
+
+def _set_listen_address(config_dir: Path):
+    config_file_path = config_dir.joinpath("config.toml")
+    if not config_file_path.exists():
+        print("Error: config.toml not found")
+        sys.exit(1)
+    with open(config_file_path, "r") as input_file:
+        config_file_content = input_file.read()
+        existing_pattern = r'^laddr = "tcp://127.0.0.1:26657"'
+        replace_with = 'laddr = "tcp://0.0.0.0:26657"'
+        print(f"Replacing in: {config_file_path}")
+        config_file_content = re.sub(existing_pattern, replace_with, config_file_content, flags=re.MULTILINE)
+    with open(config_file_path, "w") as output_file:
+        output_file.write(config_file_content)
+    app_file_path = config_dir.joinpath("app.toml")
+    if not app_file_path.exists():
+        print("Error: app.toml not found")
+        sys.exit(1)
+    with open(app_file_path, "r") as input_file:
+        app_file_content = input_file.read()
+        existing_pattern1 = r'^address = "tcp://localhost:1317"'
+        replace_with1 = 'address = "tcp://0.0.0.0:1317"'
+        app_file_content = re.sub(existing_pattern1, replace_with1, app_file_content, flags=re.MULTILINE)
+        existing_pattern2 = r'^address = "localhost:9090"'
+        replace_with2 = 'address = "0.0.0.0:9090"'
+        app_file_content = re.sub(existing_pattern2, replace_with2, app_file_content, flags=re.MULTILINE)
+    with open(app_file_path, "w") as output_file:
+        output_file.write(app_file_content)
+
+
 def _phase_from_params(parameters):
     phase = SetupPhase.ILLEGAL
     if parameters.initialize_network:
@@ -145,14 +184,19 @@ def _phase_from_params(parameters):
             print("Can't supply --initialize-network or --join-network with --create-network")
             sys.exit(1)
         phase = SetupPhase.CREATE
+    elif parameters.connect_network:
+        if parameters.initialize_network or parameters.join_network:
+            print("Can't supply --initialize-network or --join-network with --connect-network")
+            sys.exit(1)
+        phase = SetupPhase.CONNECT
     return phase
 
 
 def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCommand, extra_args):
 
-    options = command_context.cluster_context.options
+    options = opts.o
 
-    currency = "stake"  # Does this need to be a parameter?
+    currency = "alnt"  # Does this need to be a parameter?
 
     if options.debug:
         print(f"parameters: {parameters}")
@@ -177,11 +221,12 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
         output, status = run_container_command(
             command_context,
             "laconicd", f"laconicd init {parameters.node_moniker} --home {laconicd_home_path_in_container}\
-                --chain-id {parameters.chain_id}", mounts)
+                --chain-id {parameters.chain_id} --default-denom {currency}", mounts)
         if options.debug:
             print(f"Command output: {output}")
 
     elif phase == SetupPhase.JOIN:
+        # In the join phase (alternative to connect) we are participating in a genesis ceremony for the chain
         if not os.path.exists(network_dir):
             print(f"Error: network directory {network_dir} doesn't exist")
             sys.exit(1)
@@ -196,7 +241,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
         output2, status2 = run_container_command(
             command_context,
             "laconicd",
-            f"laconicd add-genesis-account {parameters.key_name} 12900000000000000000000{currency}\
+            f"laconicd genesis add-genesis-account {parameters.key_name} 12900000000000000000000{currency}\
                 --home {laconicd_home_path_in_container} --keyring-backend test",
             mounts)
         if options.debug:
@@ -204,7 +249,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
         output3, status3 = run_container_command(
             command_context,
             "laconicd",
-            f"laconicd gentx  {parameters.key_name} 90000000000{currency} --home {laconicd_home_path_in_container}\
+            f"laconicd genesis gentx  {parameters.key_name} 90000000000{currency} --home {laconicd_home_path_in_container}\
                 --chain-id {chain_id} --keyring-backend test",
             mounts)
         if options.debug:
@@ -214,7 +259,28 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
             "laconicd",
             f"laconicd keys show  {parameters.key_name} -a --home {laconicd_home_path_in_container} --keyring-backend test",
             mounts)
-        print(f"Node validator address: {output4}")
+        print(f"Node account address: {output4}")
+
+    elif phase == SetupPhase.CONNECT:
+        # In the connect phase (named to not conflict with join) we are making a node that syncs a chain with existing genesis.json
+        # but not with validator role. We need this kind of node in order to bootstrap it into a validator after it syncs
+        output1, status1 = run_container_command(
+            command_context, "laconicd", f"laconicd keys add {parameters.key_name} --home {laconicd_home_path_in_container}\
+                --keyring-backend test", mounts)
+        if options.debug:
+            print(f"Command output: {output1}")
+        output2, status2 = run_container_command(
+            command_context,
+            "laconicd",
+            f"laconicd keys show  {parameters.key_name} -a --home {laconicd_home_path_in_container} --keyring-backend test",
+            mounts)
+        print(f"Node account address: {output2}")
+        output3, status3 = run_container_command(
+            command_context,
+            "laconicd",
+            f"laconicd cometbft show-validator --home {laconicd_home_path_in_container}",
+            mounts)
+        print(f"Node validator address: {output3}")
 
     elif phase == SetupPhase.CREATE:
         if not os.path.exists(network_dir):
@@ -233,32 +299,30 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
             copyfile(genesis_file_path, os.path.join(network_dir, "config", os.path.basename(genesis_file_path)))
         else:
             # We're generating the genesis file
-            if not parameters.gentx_file_list:
-                print("Error: --gentx-files must be supplied")
-                sys.exit(1)
             # First look in the supplied gentx files for the other nodes' keys
-            other_node_keys = _get_node_keys_from_gentx_files(options, parameters.gentx_file_list)
+            other_node_keys = _get_node_keys_from_gentx_files(parameters.gentx_address_list)
             # Add those keys to our genesis, with balances we determine here (why?)
             for other_node_key in other_node_keys:
                 outputk, statusk = run_container_command(
-                    command_context, "laconicd", f"laconicd add-genesis-account {other_node_key} 12900000000000000000000{currency}\
+                    command_context, "laconicd", f"laconicd genesis add-genesis-account {other_node_key} \
+                        12900000000000000000000{currency}\
                         --home {laconicd_home_path_in_container} --keyring-backend test", mounts)
             if options.debug:
                 print(f"Command output: {outputk}")
             # Copy the gentx json files into our network dir
-            _copy_gentx_files(options, network_dir, parameters.gentx_file_list)
+            _copy_gentx_files(network_dir, parameters.gentx_file_list)
             # Now we can run collect-gentxs
             output1, status1 = run_container_command(
-                command_context, "laconicd", f"laconicd collect-gentxs --home {laconicd_home_path_in_container}", mounts)
+                command_context, "laconicd", f"laconicd genesis collect-gentxs --home {laconicd_home_path_in_container}", mounts)
             if options.debug:
                 print(f"Command output: {output1}")
             print(f"Generated genesis file, please copy to other nodes as required: \
                 {os.path.join(network_dir, 'config', 'genesis.json')}")
             # Last thing, collect-gentxs puts a likely bogus set of persistent_peers in config.toml so we remove that now
-            _remove_persistent_peers(options, network_dir)
+            _remove_persistent_peers(network_dir)
         # In both cases we validate the genesis file now
         output2, status1 = run_container_command(
-            command_context, "laconicd", f"laconicd validate-genesis --home {laconicd_home_path_in_container}", mounts)
+            command_context, "laconicd", f"laconicd genesis validate-genesis --home {laconicd_home_path_in_container}", mounts)
         print(f"validate-genesis result: {output2}")
 
     else:
@@ -266,7 +330,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
         sys.exit(1)
 
 
-def create(context: DeploymentContext, extra_args):
+def create(deployment_context: DeploymentContext, extra_args):
     network_dir = extra_args[0]
     if network_dir is None:
         print("Error: --network-dir must be supplied")
@@ -285,15 +349,18 @@ def create(context: DeploymentContext, extra_args):
         sys.exit(1)
     # Copy the network directory contents into our deployment
     # TODO: change this to work with non local paths
-    deployment_config_dir = context.deployment_dir.joinpath("data", "laconicd-config")
+    deployment_config_dir = deployment_context.deployment_dir.joinpath("data", "laconicd-config")
     copytree(config_dir_path, deployment_config_dir, dirs_exist_ok=True)
     # If supplied, add the initial persistent peers to the config file
     if extra_args[1]:
         initial_persistent_peers = extra_args[1]
-        _insert_persistent_peers(context.command_context.cluster_context.options, deployment_config_dir, initial_persistent_peers)
+        _insert_persistent_peers(deployment_config_dir, initial_persistent_peers)
+    # Enable CORS headers so explorers and so on can talk to the node
+    _enable_cors(deployment_config_dir)
+    _set_listen_address(deployment_config_dir)
     # Copy the data directory contents into our deployment
     # TODO: change this to work with non local paths
-    deployment_data_dir = context.deployment_dir.joinpath("data", "laconicd-data")
+    deployment_data_dir = deployment_context.deployment_dir.joinpath("data", "laconicd-data")
     copytree(data_dir_path, deployment_data_dir, dirs_exist_ok=True)
 
 
@@ -303,7 +370,6 @@ def init(command_context: DeployCommandContext):
 
 
 def get_state(command_context: DeployCommandContext):
-    print("Here we get state")
     return State.CONFIGURED
 
 

stack_orchestrator/data/stacks/mainnet-laconic/stack.yml

@@ -2,16 +2,17 @@ version: "1.0"
 name: mainnet-laconic
 description: "Mainnet laconic node"
 repos:
-  - cerc-io/laconicd
+  - git.vdb.to/cerc-io/laconicd
-  - lirewine/debug
+  - github.com/lirewine/debug
-  - lirewine/crypto
+  - github.com/lirewine/crypto
-  - lirewine/gem
+  - github.com/lirewine/gem
-  - lirewine/sdk
+  - github.com/lirewine/sdk
-  - cerc-io/laconic-sdk
+  - git.vdb.to/cerc-io/registry-sdk
-  - cerc-io/laconic-registry-cli
+  - git.vdb.to/cerc-io/laconic-registry-cli
-  - cerc-io/laconic-console
+  - git.vdb.to/cerc-io/laconic-console
+  - git.vdb.to/cerc-io/cosmos-explorer
 npms:
-  - laconic-sdk
+  - registry-sdk
   - laconic-registry-cli
   - debug
   - crypto
@@ -23,7 +24,8 @@ containers:
   - cerc/laconic-registry-cli
   - cerc/webapp-base
   - cerc/laconic-console-host
+  - cerc/ping-pub
 pods:
   - mainnet-laconicd
   - fixturenet-laconic-console
-
+  - laconic-explorer

stack_orchestrator/data/stacks/mainnet-laconic/test/run-mainnet-laconic-test.sh

@@ -8,8 +8,11 @@ echo "Environment variables:"
 env
 # Test laconic stack
 echo "Running laconic stack test"
-# Bit of a hack, test the most recent package
+if [ "$1" == "from-path" ]; then
-TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+    TEST_TARGET_SO="laconic-so"
+else
+    TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+fi
 # Set a non-default repo dir
 export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
 echo "Testing this package: $TEST_TARGET_SO"

stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md

@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v
 
 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
 ```
 
 ### Start the deployment

stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml

@@ -2,7 +2,7 @@ version: "1.0"
 name: merkl-sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.7
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.14
 containers:
   - cerc/watcher-merkl-sushiswap-v3
 pods:

stack_orchestrator/data/stacks/monitoring/README.md

@@ -134,6 +134,29 @@ Note: Use `host.docker.internal` as host to access ports on the host machine
 
 Place the dashboard json files in grafana dashboards config directory (`monitoring-deployment/config/monitoring/grafana/dashboards`) in the deployment folder
 
+#### Graph Node Config
+
+For graph-node dashboard postgres datasource needs to be setup in `monitoring-deployment/config/monitoring/grafana/provisioning/datasources/graph-node-postgres.yml` (in deployment folder)
+
+```yml
+# graph-node-postgres.yml
+...
+datasources:
+  - name: Graph Node Postgres
+    type: postgres
+    jsonData:
+      # Set name to remote graph-node database name
+      database: graph-node
+      ...
+    # Set user to remote graph-node database username
+    user: graph-node
+    # Add URL for remote graph-node database
+    url: graph-node-db:5432
+    # Set password for graph-node database
+    secureJsonData:
+      password: 'password'
+```
+
 ### Env
 
 Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`):
@@ -156,6 +179,11 @@ Set the following env variables in the deployment env config file (`monitoring-d
   # Grafana server host URL (used in various links in alerts, etc.)
   # (Optional, default: http://localhost:3000)
   GF_SERVER_ROOT_URL=
+
+  
+  # RPC endpoint used by graph-node for upstream head metric
+  # (Optional, default: https://mainnet.infura.io/v3)
+  GRAPH_NODE_RPC_ENDPOINT=
   ```
 
 ## Start the stack

stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md

@@ -57,35 +57,35 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
     metrics_path: /metrics
     scheme: http
     static_configs:
-      - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_PORT']
+      - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_METRICS_PORT', 'AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'azimuth'
           chain: 'ethereum'
-      - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_PORT']
+      - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_METRICS_PORT', 'CENSURES_WATCHER_HOST:CENSURES_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'censures'
           chain: 'ethereum'
-      - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_PORT']
+      - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_METRICS_PORT', 'CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'claims'
           chain: 'ethereum'
-      - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_PORT']
+      - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_METRICS_PORT', 'CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'conditional_star_release'
           chain: 'ethereum'
-      - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_PORT']
+      - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_METRICS_PORT', 'DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'delegated_sending'
           chain: 'ethereum'
-      - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_PORT']
+      - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_METRICS_PORT', 'ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'ecliptic'
           chain: 'ethereum'
-      - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_PORT']
+      - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_METRICS_PORT', 'LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'linear_star_release'
           chain: 'ethereum'
-      - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_PORT']
+      - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_METRICS_PORT', 'POLLS_WATCHER_HOST:POLLS_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'polls'
           chain: 'ethereum'
@@ -95,11 +95,11 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
     metrics_path: /metrics
     scheme: http
     static_configs:
-      - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_PORT']
+      - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_METRICS_PORT', 'SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'sushiswap'
           chain: 'filecoin'
-      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_PORT']
+      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_METRICS_PORT', 'MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'merkl_sushiswap'
           chain: 'filecoin'
@@ -109,25 +109,35 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
     metrics_path: /metrics
     scheme: http
     static_configs:
-      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_PORT']
+      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_METRICS_PORT', 'AJNA_WATCHER_HOST:AJNA_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'ajna'
           chain: 'filecoin'
+
+  - job_name: graph-node
+    metrics_path: /metrics
+    scrape_interval: 30s
+    static_configs:
+      - targets: ['GRAPH_NODE_HOST:GRAPH_NODE_HOST_METRICS_PORT']
   ```
 
 Add scrape config as done above for any additional watcher to add it to the Watchers dashboard.
 
 ### Grafana alerts config
 
-Place the pre-configured watcher alerts rules in Grafana provisioning directory:
+Place the pre-configured alerts rules in Grafana provisioning directory:
 
   ```bash
+  # watcher alert rules
   cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
+
+  # subgraph alert rules
+  cp monitoring-watchers-deployment/config/monitoring/subgraph-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
   ```
 
 Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points
 
-Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/monitoring/grafana/provisioning/alerting/policies.yaml`) with appropriate object-matchers:
+Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/policies.yml`) with appropriate object-matchers:
 
   ```yml
   ...
@@ -135,7 +145,7 @@ Add corresponding routes to the notification policies config (`monitoring-watche
       - receiver: SlackNotifier
       object_matchers:
         # Add matchers below
-        - ['grafana_folder', '=', 'WatcherAlerts']
+        - ['grafana_folder', '=~', 'WatcherAlerts|SubgraphAlerts']
   ```
 
 ### Env
@@ -149,6 +159,9 @@ Set the following env variables in the deployment env config file (`monitoring-w
   # Grafana server host URL to be used
   # (Optional, default: http://localhost:3000)
   GF_SERVER_ROOT_URL=
+
+  # List of subgraph ids to configure alerts for (separated by |)
+  CERC_GRAFANA_ALERTS_SUBGRAPH_IDS=
   ```
 
 ## Start the stack