Add deployed/error status output to the state file. (#719)

* More status info
* Up default resource limits.
* Need ps

.gitea/workflows/fixturenet-eth-plugeth-test.yml

@@ -6,6 +6,8 @@ on:
     paths:
       - '!**'
       - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '2 14 * * *'
 
 # Needed until we can incorporate docker startup into the executor container
 env:

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -6,11 +6,8 @@ on:
     paths:
       - '!**'
       - '.gitea/workflows/triggers/fixturenet-laconicd-test'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
+  schedule:
+    - cron: '1 13 * * *'
 
 jobs:
   test:
@@ -47,9 +44,5 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run fixturenet-laconicd tests"
         run: ./tests/fixturenet-laconicd/run-test.sh

.gitea/workflows/test-k8s-deploy.yml

@@ -5,12 +5,14 @@ on:
     branches: '*'
     paths:
       - '!**'
-      - '.gitea/workflows/triggers/fixturenet-laconicd-test'
+      - '.gitea/workflows/triggers/test-k8s-deploy'
+      - '.gitea/workflows/test-k8s-deploy.yml'
+      - 'tests/k8s-deploy/run-deploy-test.sh'
 
 jobs:
   test:
     name: "Run deploy test suite on kind/k8s"
-    runs-on: ubuntu-22.04-with-syn-ethdb
+    runs-on: ubuntu-22.04
     steps:
       - name: "Clone project repository"
         uses: actions/checkout@v3
@@ -34,9 +36,15 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
       - name: "Install kind"
         run: ./tests/scripts/install-kind.sh
       - name: "Install Kubectl"
         run: ./tests/scripts/install-kubectl.sh
       - name: "Run k8s deployment test"
-        run: ./tests/k8s-deploy/run-deploy-test.sh
+        run: |
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/k8s-deploy/run-deploy-test.sh
+

.gitea/workflows/triggers/fixturenet-eth-plugeth-test

@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-eth-plugeth-test CI job
 trigger
+trigger

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-laconicd-test CI job
-
+Trigger
+Trigger

stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml

@@ -2,7 +2,7 @@ services:
   laconicd:
     restart: unless-stopped
     image: cerc/laconicd:local
-    entrypoint: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
+    command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
     volumes:
       # The cosmos-sdk node's database directory:
       - laconicd-data:/root/.laconicd
@@ -15,12 +15,12 @@ services:
       - "6060"
       - "26657"
       - "26656"
-      - "9473:9473"
+      - "9473"
       - "8545"
       - "8546"
       - "9090"
       - "9091"
-      - "1317:1317"
+      - "1317"
   cli:
     image: cerc/laconic-registry-cli:local
     volumes:

stack_orchestrator/data/compose/docker-compose-geojson.yml

@@ -1,8 +0,0 @@
-version: "3.2"
-
-services:
-  geojson:
-    image: cerc/geojson:local
-    restart: always
-    ports:
-      - 0.0.0.0:8080:8080

stack_orchestrator/data/container-build/cerc-birbit/build.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-# Build the birbit image
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-docker build -t cerc/birbit:local -f ${CERC_REPO_BASE_DIR}/birbit/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/birbit

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile

@@ -10,9 +10,10 @@ COPY genesis /opt/genesis
 COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
 COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
 COPY --from=ethgen /apps /apps
-RUN cd /apps/el-gen && pip3 install -r requirements.txt
+RUN cd /apps/el-gen && pip3 install --break-system-packages -r requirements.txt
 # web3==5.24.0 used by el-gen is broken on python 3.11
-RUN pip3 install --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "typing-extensions"
 
 # Build genesis config
 RUN apk add --no-cache make bash envsubst jq

stack_orchestrator/data/container-build/cerc-geojson/Dockerfile

@@ -1,11 +0,0 @@
-FROM node:14
-
-WORKDIR /app
-
-COPY . .
-
-RUN npm install && npm run build
-
-EXPOSE 8080
-
-CMD ["npm", "run", "serve"]

stack_orchestrator/data/container-build/cerc-geojson/build.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-# Build the geojson image
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-docker build -t cerc/geojson:local -f ${CERC_CONTAINER_BASE_DIR}/cerc-geojson/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/geojson.io
-#docker build -t cerc/geojson:local -f ${CERC_REPO_BASE_DIR}/geojson.io/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/geojson.io

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -30,7 +30,7 @@ RUN \
 
 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base
+    && apt-get -y install --no-install-recommends jq gettext-base procps
 
 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"

stack_orchestrator/data/stacks/birbit/README.md

@@ -1,3 +0,0 @@
-# birbit
-
-takes geo-tagged images, publishes the metadata on chain, then generates a geojson front end of that geo data

stack_orchestrator/data/stacks/birbit/stack.yml

@@ -1,19 +0,0 @@
-version: "0.1"
-name: birbit
-repos:
-  #- github.com/zramsay/birbit
-  - github.com/mapbox/geojson.io
-containers:
-  #- cerc/birbit
-  - cerc/geojson
-pods:
-    # configurable process that reads from chain and generates a view for the front end
-    # for MVP is basically does nothing, b/c FE view is  reading all records
-    # imagine a 'watcher' for a specific view
-    #- birbit
-    # front end, gets its file from kubo;
-    # https://github.com/mapbox/geojson.io/blob/main/API.md#datadatatextx-url
-    # deployed via kubernetes
-  - geojson
-    # hosts all the dot geojson files that are required to generated views
-  - kubo

stack_orchestrator/deploy/k8s/cluster_info.py

@@ -195,7 +195,7 @@ class ClusterInfo:
                     volume_mounts=volume_mounts,
                     resources=client.V1ResourceRequirements(
                         requests={"cpu": "100m", "memory": "200Mi"},
-                        limits={"cpu": "500m", "memory": "500Mi"},
+                        limits={"cpu": "1000m", "memory": "2000Mi"},
                     ),
                 )
                 containers.append(container)

stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py

@@ -59,8 +59,8 @@ def process_app_deployment_request(
     dns_record = laconic.get_record(dns_crn)
     if dns_record:
         matched_owner = match_owner(app_deployment_request, dns_record)
-        if not matched_owner and dns_record.request:
-            matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.request, require=True))
+        if not matched_owner and dns_record.attributes.request:
+            matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.attributes.request, require=True))
 
         if matched_owner:
             print("Matched DnsRecord ownership:", matched_owner)
@@ -136,13 +136,17 @@ def load_known_requests(filename):
     return {}
 
 
-def dump_known_requests(filename, requests):
+def dump_known_requests(filename, requests, status="SEEN"):
     if not filename:
         return
     known_requests = load_known_requests(filename)
     for r in requests:
-        known_requests[r.id] = r.createTime
-    json.dump(known_requests, open(filename, "w"))
+        known_requests[r.id] = {
+            "createTime": r.createTime,
+            "status": status
+        }
+    with open(filename, "w") as f:
+        json.dump(known_requests, f)
 
 
 @click.command()
@@ -201,6 +205,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
     requests.reverse()
     requests_by_name = {}
     for r in requests:
+        # TODO: Do this _after_ filtering deployments and cancellations to minimize round trips.
         app = laconic.get_record(r.attributes.application)
         if not app:
             print("Skipping request %s, cannot locate app." % r.id)
@@ -256,6 +261,8 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
 
     if not dry_run:
         for r in requests_to_execute:
+            dump_known_requests(state_file, [r], "DEPLOYING")
+            status = "ERROR"
             try:
                 process_app_deployment_request(
                     ctx,
@@ -268,5 +275,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
                     kube_config,
                     image_registry
                 )
+                status = "DEPLOYED"
             finally:
-                dump_known_requests(state_file, [r])
+                dump_known_requests(state_file, [r], status)

stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py

@@ -40,8 +40,8 @@ def process_app_removal_request(ctx,
     matched_owner = match_owner(app_removal_request, deployment_record, dns_record)
 
     # Or of the original deployment request.
-    if not matched_owner and deployment_record.request:
-        matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.request, require=True))
+    if not matched_owner and deployment_record.attributes.request:
+        matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.attributes.request, require=True))
 
     if matched_owner:
         print("Matched deployment ownership:", matched_owner)

tests/k8s-deploy/run-deploy-test.sh

@@ -9,7 +9,7 @@ fi
 
 # Helper functions: TODO move into a separate file
 wait_for_pods_started () {
-    for i in {1..5}
+    for i in {1..50}
     do
         local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps )
 
@@ -27,7 +27,7 @@ wait_for_pods_started () {
 }
 
 wait_for_log_output () {
-    for i in {1..5}
+    for i in {1..50}
     do
 
         local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
@@ -120,7 +120,8 @@ fi
 # Stop then start again and check the volume was preserved
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop
 # Sleep a bit just in case
-sleep 2
+# sleep for longer to check if that's why the subsequent create cluster fails
+sleep 20
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
 wait_for_pods_started
 wait_for_log_output