k8s test fixes (#713)

* Add cgroup setup, increase test timeouts

* Trigger from test script or CI job changes too

.gitea/workflows/fixturenet-eth-plugeth-test.yml

@@ -6,6 +6,8 @@ on:
     paths:
       - '!**'
       - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '2 14 * * *'
 
 # Needed until we can incorporate docker startup into the executor container
 env:

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -6,11 +6,8 @@ on:
     paths:
       - '!**'
       - '.gitea/workflows/triggers/fixturenet-laconicd-test'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
+  schedule:
+    - cron: '1 13 * * *'
 
 jobs:
   test:
@@ -47,9 +44,5 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run fixturenet-laconicd tests"
         run: ./tests/fixturenet-laconicd/run-test.sh

.gitea/workflows/test-k8s-deploy.yml

@@ -5,12 +5,14 @@ on:
     branches: '*'
     paths:
       - '!**'
-      - '.gitea/workflows/triggers/fixturenet-laconicd-test'
+      - '.gitea/workflows/triggers/test-k8s-deploy'
+      - '.gitea/workflows/test-k8s-deploy.yml'
+      - 'tests/k8s-deploy/run-deploy-test.sh'
 
 jobs:
   test:
     name: "Run deploy test suite on kind/k8s"
-    runs-on: ubuntu-22.04-with-syn-ethdb
+    runs-on: ubuntu-22.04
     steps:
       - name: "Clone project repository"
         uses: actions/checkout@v3
@@ -39,4 +41,7 @@ jobs:
       - name: "Install Kubectl"
         run: ./tests/scripts/install-kubectl.sh
       - name: "Run k8s deployment test"
-        run: ./tests/k8s-deploy/run-deploy-test.sh
+        run: |
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/k8s-deploy/run-deploy-test.sh

.gitea/workflows/triggers/fixturenet-eth-plugeth-test

@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-eth-plugeth-test CI job
 trigger
+trigger

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-laconicd-test CI job
-
+Trigger
+Trigger

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile

@@ -10,9 +10,10 @@ COPY genesis /opt/genesis
 COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
 COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
 COPY --from=ethgen /apps /apps
-RUN cd /apps/el-gen && pip3 install -r requirements.txt
+RUN cd /apps/el-gen && pip3 install --break-system-packages -r requirements.txt
 # web3==5.24.0 used by el-gen is broken on python 3.11
-RUN pip3 install --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "typing-extensions"
 
 # Build genesis config
 RUN apk add --no-cache make bash envsubst jq

tests/k8s-deploy/run-deploy-test.sh

@@ -9,7 +9,7 @@ fi
 
 # Helper functions: TODO move into a separate file
 wait_for_pods_started () {
-    for i in {1..5}
+    for i in {1..50}
     do
         local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps )
 
@@ -27,7 +27,7 @@ wait_for_pods_started () {
 }
 
 wait_for_log_output () {
-    for i in {1..5}
+    for i in {1..50}
     do
 
         local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )