Add deployed/error status output to the state file. (#719 )

* More status info * Up default resource limits. * Need ps
Fix DnsRecord ownership check. (#718 )
2024-01-30 22:13:45 -06:00 · 2024-01-30 13:31:59 -06:00 · 2024-01-30 11:18:08 -07:00 · 2024-01-28 16:21:39 -07:00 · 2024-01-25 11:46:50 -07:00 · 2024-01-25 11:42:07 -07:00
19 changed files with 77 additions and 92 deletions
@@ -6,6 +6,8 @@ on:
    paths:
      - '!**'
      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '2 14 * * *'

 # Needed until we can incorporate docker startup into the executor container
 env:
@@ -6,11 +6,8 @@ on:
    paths:
      - '!**'
      - '.gitea/workflows/triggers/fixturenet-laconicd-test'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
+  schedule:
+    - cron: '1 13 * * *'

 jobs:
  test:
@@ -47,9 +44,5 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
      - name: "Run fixturenet-laconicd tests"
        run: ./tests/fixturenet-laconicd/run-test.sh
@@ -1,23 +1,18 @@
 name: K8s Deploy Test

 on:
-  pull_request:
-    branches: '*'
  push:
-    branches:
-      - main
-      - ci-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-k8s-deploy'
+      - '.gitea/workflows/test-k8s-deploy.yml'
+      - 'tests/k8s-deploy/run-deploy-test.sh'

 jobs:
  test:
-    name: "Run deploy test suite"
-    runs-on: ubuntu-latest
+    name: "Run deploy test suite on kind/k8s"
+    runs-on: ubuntu-22.04
    steps:
      - name: "Clone project repository"
        uses: actions/checkout@v3
@@ -41,15 +36,15 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Run k8s deployment test"
        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
-      - name: "Install Go"
-        uses: actions/setup-go@v4
-        with:
-          go-version: '1.21'
-      - name: "Install Kind"
-        run: go install sigs.k8s.io/kind@v0.20.0
-      - name: "Debug Kind"
-        run: kind create cluster --retain && docker logs kind-control-plane
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/k8s-deploy/run-deploy-test.sh
+
@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-eth-plugeth-test CI job
 trigger
+trigger
@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-laconicd-test CI job
-
+Trigger
+Trigger
@@ -0,0 +1 @@
+Change this file to trigger running the test-k8s-deploy CI job
@@ -2,8 +2,10 @@ version: "3.7"

 services:
  grafana:
-    image: grafana/grafana
+    image: grafana/grafana:10.2.2
    restart: always
+    environment:
+      GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
    volumes:
      - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
      - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards
@@ -2,7 +2,7 @@ version: "3.7"

 services:
  prometheus:
-    image: prom/prometheus
+    image: prom/prometheus:v2.49.1
    restart: always
    volumes:
      - ../config/monitoring/prometheus:/etc/prometheus
@@ -95,14 +95,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: censures_diff_external
@@ -191,14 +187,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: claims_diff_external
@@ -287,14 +279,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: conditional_star_release_diff_external
@@ -383,14 +371,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: delegated_sending_diff_external
@@ -479,14 +463,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: ecliptic_diff_external
@@ -575,14 +555,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: linear_star_release_diff_external
@@ -671,14 +647,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: polls_diff_external
@@ -767,14 +739,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false

@@ -865,14 +833,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: merkl_sushiswap_diff_external
@@ -961,13 +925,9 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
@@ -10,9 +10,10 @@ COPY genesis /opt/genesis
 COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
 COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
 COPY --from=ethgen /apps /apps
-RUN cd /apps/el-gen && pip3 install -r requirements.txt
+RUN cd /apps/el-gen && pip3 install --break-system-packages -r requirements.txt
 # web3==5.24.0 used by el-gen is broken on python 3.11
-RUN pip3 install --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "typing-extensions"

 # Build genesis config
 RUN apk add --no-cache make bash envsubst jq
@@ -30,7 +30,7 @@ RUN \

 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base
+    && apt-get -y install --no-install-recommends jq gettext-base procps

 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"
@@ -140,6 +140,10 @@ Set the following env variables in the deployment env config file (`monitoring-d
  # External ETH RPC endpoint (filecoin)
  # (Optional, default: https://api.node.glif.io/rpc/v1)
  CERC_FIL_RPC_ENDPOINT=
+
+  # Grafana server host URL (used in various links in alerts, etc.)
+  # (Optional, default: http://localhost:3000)
+  GF_SERVER_ROOT_URL=
  ```

 ## Start the stack
@@ -130,6 +130,10 @@ Set the following env variables in the deployment env config file (`monitoring-w
  ```bash
  # Infura key to be used
  CERC_INFURA_KEY=
+
+  # Grafana server host URL to be used
+  # (Optional, default: http://localhost:3000)
+  GF_SERVER_ROOT_URL=
  ```

 ## Start the stack
@@ -195,7 +195,7 @@ class ClusterInfo:
                    volume_mounts=volume_mounts,
                    resources=client.V1ResourceRequirements(
                        requests={"cpu": "100m", "memory": "200Mi"},
-                        limits={"cpu": "500m", "memory": "500Mi"},
+                        limits={"cpu": "1000m", "memory": "2000Mi"},
                    ),
                )
                containers.append(container)
@@ -59,8 +59,8 @@ def process_app_deployment_request(
    dns_record = laconic.get_record(dns_crn)
    if dns_record:
        matched_owner = match_owner(app_deployment_request, dns_record)
-        if not matched_owner and dns_record.request:
-            matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.request, require=True))
+        if not matched_owner and dns_record.attributes.request:
+            matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.attributes.request, require=True))

        if matched_owner:
            print("Matched DnsRecord ownership:", matched_owner)
@@ -136,13 +136,17 @@ def load_known_requests(filename):
    return {}


-def dump_known_requests(filename, requests):
+def dump_known_requests(filename, requests, status="SEEN"):
    if not filename:
        return
    known_requests = load_known_requests(filename)
    for r in requests:
-        known_requests[r.id] = r.createTime
-    json.dump(known_requests, open(filename, "w"))
+        known_requests[r.id] = {
+            "createTime": r.createTime,
+            "status": status
+        }
+    with open(filename, "w") as f:
+        json.dump(known_requests, f)


@click.command()
@@ -201,6 +205,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
    requests.reverse()
    requests_by_name = {}
    for r in requests:
+        # TODO: Do this _after_ filtering deployments and cancellations to minimize round trips.
        app = laconic.get_record(r.attributes.application)
        if not app:
            print("Skipping request %s, cannot locate app." % r.id)
@@ -256,6 +261,8 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_

    if not dry_run:
        for r in requests_to_execute:
+            dump_known_requests(state_file, [r], "DEPLOYING")
+            status = "ERROR"
            try:
                process_app_deployment_request(
                    ctx,
@@ -268,5 +275,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
                    kube_config,
                    image_registry
                )
+                status = "DEPLOYED"
            finally:
-                dump_known_requests(state_file, [r])
+                dump_known_requests(state_file, [r], status)
@@ -40,8 +40,8 @@ def process_app_removal_request(ctx,
    matched_owner = match_owner(app_removal_request, deployment_record, dns_record)

    # Or of the original deployment request.
-    if not matched_owner and deployment_record.request:
-        matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.request, require=True))
+    if not matched_owner and deployment_record.attributes.request:
+        matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.attributes.request, require=True))

    if matched_owner:
        print("Matched deployment ownership:", matched_owner)
@@ -9,7 +9,7 @@ fi

 # Helper functions: TODO move into a separate file
 wait_for_pods_started () {
-    for i in {1..5}
+    for i in {1..50}
    do
        local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps )

@@ -27,7 +27,7 @@ wait_for_pods_started () {
 }

 wait_for_log_output () {
-    for i in {1..5}
+    for i in {1..50}
    do

        local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
@@ -63,7 +63,9 @@ echo "Version reported is: ${reported_version_string}"
 echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
 rm -rf $CERC_REPO_BASE_DIR
 mkdir -p $CERC_REPO_BASE_DIR
-# Test basic stack-orchestrator deploy
+$TEST_TARGET_SO --stack test setup-repositories
+$TEST_TARGET_SO --stack test build-containers
+# Test basic stack-orchestrator deploy to k8s
 test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
 test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml
 $TEST_TARGET_SO --stack test deploy --deploy-to k8s-kind init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED
@@ -118,7 +120,8 @@ fi
 # Stop then start again and check the volume was preserved
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop
 # Sleep a bit just in case
-sleep 2
+# sleep for longer to check if that's why the subsequent create cluster fails
+sleep 20
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
 wait_for_pods_started
 wait_for_log_output
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# TODO: handle ARM
+curl --silent -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
+chmod +x ./kind
+mv ./kind /usr/local/bin
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# TODO: handle ARM
+curl --silent -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+chmod +x ./kubectl
+mv ./kubectl /usr/local/bin
Author	SHA1	Message	Date
telackey	62af03077f	Add deployed/error status output to the state file. (#719 ) * More status info * Up default resource limits. * Need ps	2024-01-30 22:13:45 -06:00
telackey	428b05158e	Fix DnsRecord ownership check. (#718 ) * Fix DnsRecord ownership check. * Var names	2024-01-30 13:31:59 -06:00
dboreham	a750b645b9	Merge Ci test branch fixes (#717 )	2024-01-30 11:18:08 -07:00
dboreham	b7f215d9bf	k8s test fixes (#713 ) * Add cgroup setup, increase test timeouts * Trigger from test script or CI job changes too	2024-01-28 16:21:39 -07:00
dboreham	eca52b10b7	Fix copy-paste error	2024-01-25 11:46:50 -07:00
dboreham	b9128841e4	Switch test back to stock runner	2024-01-25 11:42:07 -07:00
dboreham	0a302ea555	Add a schedule for fixturenet-eth-plugeth-test	2024-01-25 07:33:44 -07:00
dboreham	aa0f60baa1	Trigger fixturenet-eth-plugeth-test run	2024-01-25 06:46:18 -07:00
dboreham	cef73d8de2	Update fixturenet-laconicd-test schedule	2024-01-23 10:32:44 -07:00
dboreham	7d0f2adb46	Enable scheduled test run (#711 )	2024-01-23 09:42:24 -07:00
dboreham	5fdee25dc1	update laconicd test (#710 ) * Remove legacy docker config * Trigger test run	2024-01-23 09:25:02 -07:00
telackey	554f05de87	Fix pip3 error (#709 )	2024-01-22 21:06:15 -06:00
dboreham	b4fbee9b13	Update fixturenet-laconicd-test	2024-01-22 07:43:38 -07:00
dboreham	f826f50c4d	Merge branch 'main' of github.com:cerc-io/stack-orchestrator	2024-01-17 08:27:38 -07:00
dboreham	b83465767d	Set k8s deploy test to manual trigger	2024-01-17 08:27:09 -07:00
dboreham	50509203d1	Set k8s deploy test to manual trigger	2024-01-17 08:26:51 -07:00
prathamesh0	282e175566	Remove unnecessary hyperlinks and pin image versions (#706 ) * Remove invalid dashboard and panel ids from alert rules * Pin grafana and prometheus versions * Configure custom grafana server URL	2024-01-17 14:02:10 +05:30
dboreham	635aa7037b	Build test container	2024-01-16 21:15:21 -07:00
dboreham	9877cfaf85	Update for new runner	2024-01-16 20:08:32 -07:00
dboreham	c642e5d490	Try different runner	2024-01-16 17:09:58 -07:00
dboreham	02c49d66f5	Add debug output to check container	2024-01-16 17:07:03 -07:00
dboreham	90cebdb7a6	Add CI script for k8s deployment test (#705 )	2024-01-16 16:16:07 -07:00
				`@@ -0,0 +1 @@`
				`Change this file to trigger running the test-k8s-deploy CI job`