Merge branch 'main' into telackey/sostatus

.gitea/workflows/fixturenet-eth-plugeth-test.yml

@@ -6,6 +6,8 @@ on:
     paths:
       - '!**'
       - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '2 14 * * *'
 
 # Needed until we can incorporate docker startup into the executor container
 env:

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -6,11 +6,8 @@ on:
     paths:
       - '!**'
       - '.gitea/workflows/triggers/fixturenet-laconicd-test'
-
+  schedule:
-# Needed until we can incorporate docker startup into the executor container
+    - cron: '1 13 * * *'
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
 
 jobs:
   test:
@@ -47,9 +44,5 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run fixturenet-laconicd tests"
         run: ./tests/fixturenet-laconicd/run-test.sh

.gitea/workflows/test-k8s-deploy.yml

@@ -1,23 +1,18 @@
 name: K8s Deploy Test
 
 on:
-  pull_request:
-    branches: '*'
   push:
-    branches:
+    branches: '*'
-      - main
+    paths:
-      - ci-test
+      - '!**'
-    paths-ignore:
+      - '.gitea/workflows/triggers/test-k8s-deploy'
-      - '.gitea/workflows/triggers/*'
+      - '.gitea/workflows/test-k8s-deploy.yml'
-
+      - 'tests/k8s-deploy/run-deploy-test.sh'
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
 
 jobs:
   test:
     name: "Run deploy test suite on kind/k8s"
-    runs-on: ubuntu-22.04-with-syn-ethdb
+    runs-on: ubuntu-22.04
     steps:
       - name: "Clone project repository"
         uses: actions/checkout@v3
@@ -41,14 +36,15 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
+      - name: "Check cgroups version"
-        run: |
+        run: mount | grep cgroup
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-      - name: "Check container"
-        run: cat /usr/local/bin/entrypoint.sh
       - name: "Install kind"
         run: ./tests/scripts/install-kind.sh
       - name: "Install Kubectl"
         run: ./tests/scripts/install-kubectl.sh
       - name: "Run k8s deployment test"
-        run: ./tests/k8s-deploy/run-deploy-test.sh
+        run: |
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/k8s-deploy/run-deploy-test.sh
+

.gitea/workflows/triggers/fixturenet-eth-plugeth-test

@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-eth-plugeth-test CI job
 trigger
+trigger

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-laconicd-test CI job
-
+Trigger
+Trigger

.gitea/workflows/triggers/test-k8s-deploy

@@ -0,0 +1 @@
+Change this file to trigger running the test-k8s-deploy CI job

stack_orchestrator/data/compose/docker-compose-grafana.yml

@@ -2,8 +2,10 @@ version: "3.7"
 
 services:
   grafana:
-    image: grafana/grafana
+    image: grafana/grafana:10.2.2
     restart: always
+    environment:
+      GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
     volumes:
       - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
       - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards

stack_orchestrator/data/compose/docker-compose-prom-server.yml

@@ -2,7 +2,7 @@ version: "3.7"
 
 services:
   prometheus:
-    image: prom/prometheus
+    image: prom/prometheus:v2.49.1
     restart: always
     volumes:
       - ../config/monitoring/prometheus:/etc/prometheus

stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml

@@ -95,14 +95,10 @@ groups:
               maxDataPoints: 43200
               refId: condition
               type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
         noDataState: Alerting
         execErrState: Alerting
         for: 15m
         annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
       - uid: censures_diff_external
@@ -191,14 +187,10 @@ groups:
               maxDataPoints: 43200
               refId: condition
               type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
         noDataState: Alerting
         execErrState: Alerting
         for: 15m
         annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
       - uid: claims_diff_external
@@ -287,14 +279,10 @@ groups:
               maxDataPoints: 43200
               refId: condition
               type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
         noDataState: Alerting
         execErrState: Alerting
         for: 15m
         annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
       - uid: conditional_star_release_diff_external
@@ -383,14 +371,10 @@ groups:
               maxDataPoints: 43200
               refId: condition
               type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
         noDataState: Alerting
         execErrState: Alerting
         for: 15m
         annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
       - uid: delegated_sending_diff_external
@@ -479,14 +463,10 @@ groups:
               maxDataPoints: 43200
               refId: condition
               type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
         noDataState: Alerting
         execErrState: Alerting
         for: 15m
         annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
       - uid: ecliptic_diff_external
@@ -575,14 +555,10 @@ groups:
               maxDataPoints: 43200
               refId: condition
               type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
         noDataState: Alerting
         execErrState: Alerting
         for: 15m
         annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
       - uid: linear_star_release_diff_external
@@ -671,14 +647,10 @@ groups:
               maxDataPoints: 43200
               refId: condition
               type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
         noDataState: Alerting
         execErrState: Alerting
         for: 15m
         annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
       - uid: polls_diff_external
@@ -767,14 +739,10 @@ groups:
               maxDataPoints: 43200
               refId: condition
               type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
         noDataState: Alerting
         execErrState: Alerting
         for: 15m
         annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
 
@@ -865,14 +833,10 @@ groups:
               maxDataPoints: 43200
               refId: condition
               type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
         noDataState: Alerting
         execErrState: Alerting
         for: 15m
         annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
       - uid: merkl_sushiswap_diff_external
@@ -961,13 +925,9 @@ groups:
               maxDataPoints: 43200
               refId: condition
               type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
         noDataState: Alerting
         execErrState: Alerting
         for: 15m
         annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile

@@ -10,9 +10,10 @@ COPY genesis /opt/genesis
 COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
 COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
 COPY --from=ethgen /apps /apps
-RUN cd /apps/el-gen && pip3 install -r requirements.txt
+RUN cd /apps/el-gen && pip3 install --break-system-packages -r requirements.txt
 # web3==5.24.0 used by el-gen is broken on python 3.11
-RUN pip3 install --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "typing-extensions"
 
 # Build genesis config
 RUN apk add --no-cache make bash envsubst jq

stack_orchestrator/data/stacks/monitoring/README.md

@@ -140,6 +140,10 @@ Set the following env variables in the deployment env config file (`monitoring-d
   # External ETH RPC endpoint (filecoin)
   # (Optional, default: https://api.node.glif.io/rpc/v1)
   CERC_FIL_RPC_ENDPOINT=
+
+  # Grafana server host URL (used in various links in alerts, etc.)
+  # (Optional, default: http://localhost:3000)
+  GF_SERVER_ROOT_URL=
   ```
 
 ## Start the stack

stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md

@@ -130,6 +130,10 @@ Set the following env variables in the deployment env config file (`monitoring-w
   ```bash
   # Infura key to be used
   CERC_INFURA_KEY=
+
+  # Grafana server host URL to be used
+  # (Optional, default: http://localhost:3000)
+  GF_SERVER_ROOT_URL=
   ```
 
 ## Start the stack

tests/k8s-deploy/run-deploy-test.sh

@@ -9,7 +9,7 @@ fi
 
 # Helper functions: TODO move into a separate file
 wait_for_pods_started () {
-    for i in {1..5}
+    for i in {1..50}
     do
         local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps )
 
@@ -27,7 +27,7 @@ wait_for_pods_started () {
 }
 
 wait_for_log_output () {
-    for i in {1..5}
+    for i in {1..50}
     do
 
         local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
@@ -63,7 +63,9 @@ echo "Version reported is: ${reported_version_string}"
 echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
 rm -rf $CERC_REPO_BASE_DIR
 mkdir -p $CERC_REPO_BASE_DIR
-# Test basic stack-orchestrator deploy
+$TEST_TARGET_SO --stack test setup-repositories
+$TEST_TARGET_SO --stack test build-containers
+# Test basic stack-orchestrator deploy to k8s
 test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
 test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml
 $TEST_TARGET_SO --stack test deploy --deploy-to k8s-kind init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED
@@ -118,7 +120,8 @@ fi
 # Stop then start again and check the volume was preserved
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop
 # Sleep a bit just in case
-sleep 2
+# sleep for longer to check if that's why the subsequent create cluster fails
+sleep 20
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
 wait_for_pods_started
 wait_for_log_output