Fix DnsRecord ownership check. (#718 )

* Fix DnsRecord ownership check. * Var names
Merge Ci test branch fixes (#717 )
2024-01-30 13:31:59 -06:00 · 2024-01-30 11:18:08 -07:00 · 2024-01-28 16:21:39 -07:00 · 2024-01-25 11:46:50 -07:00 · 2024-01-25 11:42:07 -07:00 · 2024-01-25 07:33:44 -07:00
28 changed files with 190 additions and 129 deletions
@@ -6,6 +6,8 @@ on:
    paths:
      - '!**'
      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '2 14 * * *'

 # Needed until we can incorporate docker startup into the executor container
 env:
@@ -6,11 +6,8 @@ on:
    paths:
      - '!**'
      - '.gitea/workflows/triggers/fixturenet-laconicd-test'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
+  schedule:
+    - cron: '1 13 * * *'

 jobs:
  test:
@@ -47,9 +44,5 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
      - name: "Run fixturenet-laconicd tests"
        run: ./tests/fixturenet-laconicd/run-test.sh
@@ -1,23 +1,18 @@
 name: K8s Deploy Test

 on:
-  pull_request:
-    branches: '*'
  push:
-    branches:
-      - main
-      - ci-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-k8s-deploy'
+      - '.gitea/workflows/test-k8s-deploy.yml'
+      - 'tests/k8s-deploy/run-deploy-test.sh'

 jobs:
  test:
-    name: "Run deploy test suite"
-    runs-on: ubuntu-latest
+    name: "Run deploy test suite on kind/k8s"
+    runs-on: ubuntu-22.04
    steps:
      - name: "Clone project repository"
        uses: actions/checkout@v3
@@ -41,15 +36,15 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Run k8s deployment test"
        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
-      - name: "Install Go"
-        uses: actions/setup-go@v4
-        with:
-          go-version: '1.21'
-      - name: "Install Kind"
-        run: go install sigs.k8s.io/kind@v0.20.0
-      - name: "Debug Kind"
-        run: kind create cluster --retain && docker logs kind-control-plane
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/k8s-deploy/run-deploy-test.sh
+
@@ -41,6 +41,8 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
+      - name: "Install wget" # 20240109 - Only needed until the executors are updated.
+        run: apt update && apt install -y wget
      - name: Start dockerd # Also needed until we can incorporate into the executor
        run: |
          dockerd -H $DOCKER_HOST --userland-proxy=false &
@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-eth-plugeth-test CI job
 trigger
+trigger
@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-laconicd-test CI job
-
+Trigger
+Trigger
@@ -0,0 +1 @@
+Change this file to trigger running the test-k8s-deploy CI job
@@ -2,8 +2,10 @@ version: "3.7"

 services:
  grafana:
-    image: grafana/grafana
+    image: grafana/grafana:10.2.2
    restart: always
+    environment:
+      GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
    volumes:
      - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
      - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards
@@ -6,6 +6,7 @@ services:
    restart: on-failure
    environment:
      - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL}
+      - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438
      - NEXT_PUBLIC_BASEPATH=/apps/osmosis
      - NEXT_PUBLIC_URBIT_DEPLOYMENT=true
    working_dir: /app/packages/web
@@ -6,6 +6,7 @@ services:
    restart: on-failure
    environment:
      - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL}
+      - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438
    working_dir: /app/packages/web
    command: ["./build-app.sh"]
    volumes:
@@ -2,7 +2,7 @@ version: "3.7"

 services:
  prometheus:
-    image: prom/prometheus
+    image: prom/prometheus:v2.49.1
    restart: always
    volumes:
      - ../config/monitoring/prometheus:/etc/prometheus
@@ -95,14 +95,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: censures_diff_external
@@ -191,14 +187,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: claims_diff_external
@@ -287,14 +279,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: conditional_star_release_diff_external
@@ -383,14 +371,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: delegated_sending_diff_external
@@ -479,14 +463,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: ecliptic_diff_external
@@ -575,14 +555,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: linear_star_release_diff_external
@@ -671,14 +647,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: polls_diff_external
@@ -767,14 +739,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false

@@ -865,14 +833,10 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
      - uid: merkl_sushiswap_diff_external
@@ -961,13 +925,9 @@ groups:
              maxDataPoints: 43200
              refId: condition
              type: math
-        dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca
-        panelId: 24
        noDataState: Alerting
        execErrState: Alerting
        for: 15m
        annotations:
-          __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca
-          __panelId__: "24"
          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
        isPaused: false
@@ -2,4 +2,6 @@
 # Build a local version of the task executor for act-runner
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/act-runner/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR}
+
+cd ${CERC_REPO_BASE_DIR}/hosting/act-runner
+docker build -t cerc/act-runner-task-executor:local -f Dockerfile.task-executor ${build_command_args} .
@@ -10,9 +10,10 @@ COPY genesis /opt/genesis
 COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
 COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
 COPY --from=ethgen /apps /apps
-RUN cd /apps/el-gen && pip3 install -r requirements.txt
+RUN cd /apps/el-gen && pip3 install --break-system-packages -r requirements.txt
 # web3==5.24.0 used by el-gen is broken on python 3.11
-RUN pip3 install --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "typing-extensions"

 # Build genesis config
 RUN apk add --no-cache make bash envsubst jq
@@ -21,6 +21,11 @@ WORK_DIR="${1:-/app}"

 cd "${WORK_DIR}" || exit 1

+# If this file doesn't exist at all, we'll get errors below.
+if [ ! -f "next.config.js" ]; then
+  touch next.config.js
+fi
+
 if [ ! -f "next.config.dist" ]; then
  cp next.config.js next.config.dist
 fi
@@ -3,7 +3,7 @@ name: act-runner
 description: "Local act-runner"
 repos:
  - git.vdb.to/cerc-io/hosting
-  - gitea.com/gitea/act_runner
+  - gitea.com/telackey/act_runner@telackey/entrypoint
 containers:
  - cerc/act-runner
  - cerc/act-runner-task-executor
@@ -140,6 +140,10 @@ Set the following env variables in the deployment env config file (`monitoring-d
  # External ETH RPC endpoint (filecoin)
  # (Optional, default: https://api.node.glif.io/rpc/v1)
  CERC_FIL_RPC_ENDPOINT=
+
+  # Grafana server host URL (used in various links in alerts, etc.)
+  # (Optional, default: http://localhost:3000)
+  GF_SERVER_ROOT_URL=
  ```

 ## Start the stack
@@ -130,6 +130,10 @@ Set the following env variables in the deployment env config file (`monitoring-w
  ```bash
  # Infura key to be used
  CERC_INFURA_KEY=
+
+  # Grafana server host URL to be used
+  # (Optional, default: http://localhost:3000)
+  GF_SERVER_ROOT_URL=
  ```

 ## Start the stack
@@ -3,7 +3,7 @@ name: package-registry
 description: "Local Package Registry"
 repos:
  - git.vdb.to/cerc-io/hosting
-  - gitea.com/gitea/act_runner
+  - gitea.com/telackey/act_runner@telackey/entrypoint
 containers:
  - cerc/act-runner
  - cerc/act-runner-task-executor
@@ -444,7 +444,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
    deployer_config_generator = getDeployerConfigGenerator(deployment_type)
    # TODO: make deployment_dir_path a Path above
    deployer_config_generator.generate(deployment_dir_path)
-    call_stack_deploy_create(deployment_context, [network_dir, initial_peers])
+    call_stack_deploy_create(deployment_context, [network_dir, initial_peers, deployment_command_context])


 # TODO: this code should be in the stack .py files but
@@ -168,8 +168,8 @@ class ClusterInfo:
            result.append(pv)
        return result

-    # to suit the deployment, and also annotate the container specs to point at said volumes
-    def get_deployment(self):
+    # TODO: put things like image pull policy into an object-scope struct
+    def get_deployment(self, image_pull_policy: str = None):
        containers = []
        for pod_name in self.parsed_pod_yaml_map:
            pod = self.parsed_pod_yaml_map[pod_name]
@@ -189,7 +189,7 @@ class ClusterInfo:
                container = client.V1Container(
                    name=container_name,
                    image=image_to_use,
-                    image_pull_policy="Always",
+                    image_pull_policy=image_pull_policy,
                    env=envs_from_environment_variables_map(self.environment_variables.map),
                    ports=[client.V1ContainerPort(container_port=port)],
                    volume_mounts=volume_mounts,
@@ -111,7 +111,7 @@ class K8sDeployer(Deployer):
                print("PVCs created:")
                print(f"{pvc_resp}")
        # Process compose files into a Deployment
-        deployment = self.cluster_info.get_deployment()
+        deployment = self.cluster_info.get_deployment(image_pull_policy=None if self.is_kind() else "Always")
        # Create the k8s objects
        if opts.o.debug:
            print(f"Sending this deployment: {deployment}")
@@ -132,18 +132,18 @@ class K8sDeployer(Deployer):
            print("Service created:")
            print(f"{service_resp}")

-        # TODO: disable ingress for kind
-        ingress: client.V1Ingress = self.cluster_info.get_ingress()
+        if not self.is_kind():
+            ingress: client.V1Ingress = self.cluster_info.get_ingress()

-        if opts.o.debug:
-            print(f"Sending this ingress: {ingress}")
-        ingress_resp = self.networking_api.create_namespaced_ingress(
-            namespace=self.k8s_namespace,
-            body=ingress
-        )
-        if opts.o.debug:
-            print("Ingress created:")
-            print(f"{ingress_resp}")
+            if opts.o.debug:
+                print(f"Sending this ingress: {ingress}")
+            ingress_resp = self.networking_api.create_namespaced_ingress(
+                namespace=self.k8s_namespace,
+                body=ingress
+            )
+            if opts.o.debug:
+                print("Ingress created:")
+                print(f"{ingress_resp}")

    def down(self, timeout, volumes):
        self.connect_api()
@@ -196,16 +196,16 @@ class K8sDeployer(Deployer):
        except client.exceptions.ApiException as e:
            _check_delete_exception(e)

-        # TODO: disable ingress for kind
-        ingress: client.V1Ingress = self.cluster_info.get_ingress()
-        if opts.o.debug:
-            print(f"Deleting this ingress: {ingress}")
-        try:
-            self.networking_api.delete_namespaced_ingress(
-                name=ingress.metadata.name, namespace=self.k8s_namespace
-            )
-        except client.exceptions.ApiException as e:
-            _check_delete_exception(e)
+        if not self.is_kind():
+            ingress: client.V1Ingress = self.cluster_info.get_ingress()
+            if opts.o.debug:
+                print(f"Deleting this ingress: {ingress}")
+            try:
+                self.networking_api.delete_namespaced_ingress(
+                    name=ingress.metadata.name, namespace=self.k8s_namespace
+                )
+            except client.exceptions.ApiException as e:
+                _check_delete_exception(e)

        if self.is_kind():
            # Destroy the kind cluster
@@ -219,7 +219,7 @@ class K8sDeployer(Deployer):

        if all_pods.items:
            for p in all_pods.items:
-                if self.cluster_info.app_name in p.metadata.name:
+                if f"{self.cluster_info.app_name}-deployment" in p.metadata.name:
                    pods.append(p)

        if not pods:
@@ -266,7 +266,7 @@ class K8sDeployer(Deployer):
        ret = []

        for p in pods.items:
-            if self.cluster_info.app_name in p.metadata.name:
+            if f"{self.cluster_info.app_name}-deployment" in p.metadata.name:
                pod_ip = p.status.pod_ip
                ports = AttrDict()
                for c in p.spec.containers:
@@ -299,11 +299,20 @@ class K8sDeployer(Deployer):

    def logs(self, services, tail, follow, stream):
        self.connect_api()
-        pods = pods_in_deployment(self.core_api, "test-deployment")
+        pods = pods_in_deployment(self.core_api, self.cluster_info.app_name)
        if len(pods) > 1:
            print("Warning: more than one pod in the deployment")
-        k8s_pod_name = pods[0]
-        log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
+        if len(pods) == 0:
+            log_data = "******* Pods not running ********\n"
+        else:
+            k8s_pod_name = pods[0]
+            # If the pod is not yet started, the logs request below will throw an exception
+            try:
+                log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
+            except client.exceptions.ApiException as e:
+                if opts.o.debug:
+                    print(f"Error from read_namespaced_pod_log: {e}")
+                log_data = "******* No logs available ********\n"
        return log_stream_from_string(log_data)

    def update(self):
@@ -21,6 +21,7 @@ from typing import Set, Mapping, List

 from stack_orchestrator.opts import opts
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names
+from stack_orchestrator.deploy.deployer import DeployerException


 def _run_command(command: str):
@@ -29,10 +30,13 @@ def _run_command(command: str):
    result = subprocess.run(command, shell=True)
    if opts.o.debug:
        print(f"Result: {result}")
+    return result


 def create_cluster(name: str, config_file: str):
-    _run_command(f"kind create cluster --name {name} --config {config_file}")
+    result = _run_command(f"kind create cluster --name {name} --config {config_file}")
+    if result.returncode != 0:
+        raise DeployerException(f"kind create cluster failed: {result}")


 def destroy_cluster(name: str):
@@ -41,12 +45,14 @@ def destroy_cluster(name: str):

 def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]):
    for image in image_set:
-        _run_command(f"kind load docker-image {image} --name {kind_cluster_name}")
+        result = _run_command(f"kind load docker-image {image} --name {kind_cluster_name}")
+        if result.returncode != 0:
+            raise DeployerException(f"kind create cluster failed: {result}")


 def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
    pods = []
-    pod_response = core_api.list_namespaced_pod(namespace="default", label_selector="app=test-app")
+    pod_response = core_api.list_namespaced_pod(namespace="default", label_selector=f"app={deployment_name}")
    if opts.o.debug:
        print(f"pod_response: {pod_response}")
    for pod_info in pod_response.items:
@@ -59,8 +59,8 @@ def process_app_deployment_request(
    dns_record = laconic.get_record(dns_crn)
    if dns_record:
        matched_owner = match_owner(app_deployment_request, dns_record)
-        if not matched_owner and dns_record.request:
-            matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.request, require=True))
+        if not matched_owner and dns_record.attributes.request:
+            matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.attributes.request, require=True))

        if matched_owner:
            print("Matched DnsRecord ownership:", matched_owner)
@@ -40,8 +40,8 @@ def process_app_removal_request(ctx,
    matched_owner = match_owner(app_removal_request, deployment_record, dns_record)

    # Or of the original deployment request.
-    if not matched_owner and deployment_record.request:
-        matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.request, require=True))
+    if not matched_owner and deployment_record.attributes.request:
+        matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.attributes.request, require=True))

    if matched_owner:
        print("Matched deployment ownership:", matched_owner)
@@ -1,14 +1,59 @@
 #!/usr/bin/env bash
 set -e
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-  set -x
+    set -x
+    # Dump environment variables for debugging
+    echo "Environment variables:"
+    env
 fi
+
+# Helper functions: TODO move into a separate file
+wait_for_pods_started () {
+    for i in {1..50}
+    do
+        local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps )
+
+        if [[ "$ps_output" == *"Running containers:"* ]]; then
+            # if ready, return
+            return
+        else
+            # if not ready, wait
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "waiting for pods to start: FAILED"
+    delete_cluster_exit
+}
+
+wait_for_log_output () {
+    for i in {1..50}
+    do
+
+        local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+
+        if [[ ! -z "$log_output" ]]; then
+            # if ready, return
+            return
+        else
+            # if not ready, wait
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "waiting for pods log content: FAILED"
+    delete_cluster_exit
+}
+
+
+delete_cluster_exit () {
+    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+    exit 1
+}
+
 # Note: eventually this test should be folded into ../deploy/
 # but keeping it separate for now for convenience
 TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
-# Dump environment variables for debugging
-echo "Environment variables:"
-env
 # Set a non-default repo dir
 export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
 echo "Testing this package: $TEST_TARGET_SO"
@@ -18,7 +63,9 @@ echo "Version reported is: ${reported_version_string}"
 echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
 rm -rf $CERC_REPO_BASE_DIR
 mkdir -p $CERC_REPO_BASE_DIR
-# Test basic stack-orchestrator deploy
+$TEST_TARGET_SO --stack test setup-repositories
+$TEST_TARGET_SO --stack test build-containers
+# Test basic stack-orchestrator deploy to k8s
 test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
 test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml
 $TEST_TARGET_SO --stack test deploy --deploy-to k8s-kind init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED
@@ -53,23 +100,37 @@ fi
 echo "deploy create output file test: passed"
 # Try to start the deployment
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
-# TODO: add a check to see if the container is up
-# Sleep because k8s not up yet
-sleep 30
+wait_for_pods_started
 # Check logs command works
+wait_for_log_output
 log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
 if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then
    echo "deployment logs test: passed"
 else
    echo "deployment logs test: FAILED"
-    exit 1
+    delete_cluster_exit
 fi
 # Check the config variable CERC_TEST_PARAM_1 was passed correctly
 if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then
    echo "deployment config test: passed"
 else
    echo "deployment config test: FAILED"
-    exit 1
+    delete_cluster_exit
+fi
+# Stop then start again and check the volume was preserved
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
+# Sleep a bit just in case
+# sleep for longer to check if that's why the subsequent create cluster fails
+sleep 20
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+wait_for_pods_started
+wait_for_log_output
+log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_4" == *"Filesystem is old"* ]]; then
+    echo "Retain volumes test: passed"
+else
+    echo "Retain volumes test: FAILED"
+    delete_cluster_exit
 fi
 # Stop and clean up
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# TODO: handle ARM
+curl --silent -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
+chmod +x ./kind
+mv ./kind /usr/local/bin
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# TODO: handle ARM
+curl --silent -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+chmod +x ./kubectl
+mv ./kubectl /usr/local/bin
Author	SHA1	Message	Date
telackey	428b05158e	Fix DnsRecord ownership check. (#718 ) * Fix DnsRecord ownership check. * Var names	2024-01-30 13:31:59 -06:00
dboreham	a750b645b9	Merge Ci test branch fixes (#717 )	2024-01-30 11:18:08 -07:00
dboreham	b7f215d9bf	k8s test fixes (#713 ) * Add cgroup setup, increase test timeouts * Trigger from test script or CI job changes too	2024-01-28 16:21:39 -07:00
dboreham	eca52b10b7	Fix copy-paste error	2024-01-25 11:46:50 -07:00
dboreham	b9128841e4	Switch test back to stock runner	2024-01-25 11:42:07 -07:00
dboreham	0a302ea555	Add a schedule for fixturenet-eth-plugeth-test	2024-01-25 07:33:44 -07:00
dboreham	aa0f60baa1	Trigger fixturenet-eth-plugeth-test run	2024-01-25 06:46:18 -07:00
dboreham	cef73d8de2	Update fixturenet-laconicd-test schedule	2024-01-23 10:32:44 -07:00
dboreham	7d0f2adb46	Enable scheduled test run (#711 )	2024-01-23 09:42:24 -07:00
dboreham	5fdee25dc1	update laconicd test (#710 ) * Remove legacy docker config * Trigger test run	2024-01-23 09:25:02 -07:00
telackey	554f05de87	Fix pip3 error (#709 )	2024-01-22 21:06:15 -06:00
dboreham	b4fbee9b13	Update fixturenet-laconicd-test	2024-01-22 07:43:38 -07:00
dboreham	f826f50c4d	Merge branch 'main' of github.com:cerc-io/stack-orchestrator	2024-01-17 08:27:38 -07:00
dboreham	b83465767d	Set k8s deploy test to manual trigger	2024-01-17 08:27:09 -07:00
dboreham	50509203d1	Set k8s deploy test to manual trigger	2024-01-17 08:26:51 -07:00
prathamesh0	282e175566	Remove unnecessary hyperlinks and pin image versions (#706 ) * Remove invalid dashboard and panel ids from alert rules * Pin grafana and prometheus versions * Configure custom grafana server URL	2024-01-17 14:02:10 +05:30
dboreham	635aa7037b	Build test container	2024-01-16 21:15:21 -07:00
dboreham	9877cfaf85	Update for new runner	2024-01-16 20:08:32 -07:00
dboreham	c642e5d490	Try different runner	2024-01-16 17:09:58 -07:00
dboreham	02c49d66f5	Add debug output to check container	2024-01-16 17:07:03 -07:00
dboreham	90cebdb7a6	Add CI script for k8s deployment test (#705 )	2024-01-16 16:16:07 -07:00
dboreham	1f9653e6f7	Fix kind mode and add k8s deployment test (#704 ) * Fix kind mode and add k8s deployment test * Fix lint errors	2024-01-16 15:55:58 -07:00
telackey	0587813dd0	Create next.config.js if it is missing. (#701 ) * Create next.config.js if it is missing. * Add comment.	2024-01-15 12:12:59 -06:00
telackey	4b3ea7c30f	Update independent act-runner stack to use custom act as well. (#702 ) * Update independent act-runner stack to use custom act as well. * Remove branches which are not needed or already merged.	2024-01-15 12:10:48 -06:00
nabarun	db8aec52aa	Pin commit hash of asset list repo in osmosis frontend app (#703 )	2024-01-15 16:56:06 +05:30
telackey	b83030f63b	Use custom act with gitea. (#700 ) * Use custom act with gitea. * Make sure wget is available * Fix repo url	2024-01-09 22:53:43 -06:00
				`@@ -0,0 +1 @@`
				`Change this file to trigger running the test-k8s-deploy CI job`