Basic volume support (#622)

add fixturenet-laconicd test

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -0,0 +1,55 @@
+name: Fixturenet-Laconicd-Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/fixturenet-laconicd-test'
+
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
+
+jobs:
+  test:
+    name: "Run an Laconicd fixturenet test"
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Update'
+        run: apt-get update
+      - name: 'Setup jq'
+        run: apt-get install jq -y
+      - name: 'Check jq'
+        run: |
+          which jq
+          jq --version
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: Start dockerd # Also needed until we can incorporate into the executor
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
+      - name: "Run fixturenet-laconicd tests"
+        run: ./tests/fixturenet-laconicd/run-test.sh

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -0,0 +1,2 @@
+Change this file to trigger running the fixturenet-laconicd-test CI job
+

.github/workflows/fixturenet-laconicd.yml

@@ -0,0 +1,30 @@
+name: Fixturenet-Laconicd Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.github/workflows/triggers/fixturenet-laconicd-test'
+
+jobs:
+  test:
+    name: "Run fixturenet-laconicd test suite"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Run fixturenet-laconicd tests"
+        run: ./tests/fixturenet-laconicd/run-test.sh

.github/workflows/triggers/fixturenet-laconicd-test

@@ -0,0 +1,3 @@
+Change this file to trigger running the fixturenet-laconicd-test CI job
+
+trigger

app/deploy/deployment_create.py

@@ -261,7 +261,7 @@ def init(ctx, config, output, map_ports_to_host):
     config_variables = _parse_config_variables(config)
     if config_variables:
         # Implement merge, since update() overwrites
-        orig_config = spec_file_content["config"]
+        orig_config = spec_file_content.get("config", {})
         new_config = config_variables["config"]
         merged_config = {**new_config, **orig_config}
         spec_file_content.update({"config": merged_config})

app/deploy/k8s/cluster_info.py

@@ -18,6 +18,7 @@ from typing import Any, List, Set
 
 from app.opts import opts
 from app.util import get_yaml
+from app.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files
 
 
 class ClusterInfo:
@@ -47,6 +48,28 @@ class ClusterInfo:
         if opts.o.debug:
             print(f"image_set: {self.image_set}")
 
+    def get_pvcs(self):
+        result = []
+        volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        if opts.o.debug:
+            print(f"Volumes: {volumes}")
+        for volume_name in volumes:
+            spec = client.V1PersistentVolumeClaimSpec(
+                storage_class_name="standard",
+                access_modes=["ReadWriteOnce"],
+                resources=client.V1ResourceRequirements(
+                    requests={"storage": "2Gi"}
+                )
+            )
+            pvc = client.V1PersistentVolumeClaim(
+                metadata=client.V1ObjectMeta(name=volume_name,
+                                             labels={"volume-label": volume_name}),
+                spec=spec,
+            )
+            result.append(pvc)
+        return result
+
+    # to suit the deployment, and also annotate the container specs to point at said volumes
     def get_deployment(self):
         containers = []
         for pod_name in self.parsed_pod_yaml_map:
@@ -56,19 +79,22 @@ class ClusterInfo:
                 container_name = service_name
                 service_info = services[service_name]
                 image = service_info["image"]
+                volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name)
                 container = client.V1Container(
                     name=container_name,
                     image=image,
                     ports=[client.V1ContainerPort(container_port=80)],
+                    volume_mounts=volume_mounts,
                     resources=client.V1ResourceRequirements(
                         requests={"cpu": "100m", "memory": "200Mi"},
                         limits={"cpu": "500m", "memory": "500Mi"},
                     ),
                 )
                 containers.append(container)
+        volumes = volumes_for_pod_files(self.parsed_pod_yaml_map)
         template = client.V1PodTemplateSpec(
             metadata=client.V1ObjectMeta(labels={"app": self.app_name}),
-            spec=client.V1PodSpec(containers=containers),
+            spec=client.V1PodSpec(containers=containers, volumes=volumes),
         )
         spec = client.V1DeploymentSpec(
             replicas=1, template=template, selector={

app/deploy/k8s/deploy_k8s.py

@@ -26,6 +26,7 @@ class K8sDeployer(Deployer):
     name: str = "k8s"
     core_api: client.CoreV1Api
     apps_api: client.AppsV1Api
+    k8s_namespace: str = "default"
     kind_cluster_name: str
     cluster_info : ClusterInfo
 
@@ -49,17 +50,27 @@ class K8sDeployer(Deployer):
         self.connect_api()
         # Ensure the referenced containers are copied into kind
         load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set)
+        # Figure out the PVCs for this deployment
+        pvcs = self.cluster_info.get_pvcs()
+        for pvc in pvcs:
+            if opts.o.debug:
+                print(f"Sending this: {pvc}")
+            pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace=self.k8s_namespace)
+            if opts.o.debug:
+                print("PVCs created:")
+                print(f"{pvc_resp}")
         # Process compose files into a Deployment
         deployment = self.cluster_info.get_deployment()
         # Create the k8s objects
-        resp = self.apps_api.create_namespaced_deployment(
-            body=deployment, namespace="default"
-        )
-
         if opts.o.debug:
-            print("Deployment created.\n")
-            print(f"{resp.metadata.namespace} {resp.metadata.name} \
-                  {resp.metadata.generation} {resp.spec.template.spec.containers[0].image}")
+            print(f"Sending this: {deployment}")
+        deployment_resp = self.apps_api.create_namespaced_deployment(
+            body=deployment, namespace=self.k8s_namespace
+        )
+        if opts.o.debug:
+            print("Deployment created:")
+            print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \
+                  {deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}")
 
     def down(self, timeout, volumes):
         # Delete the k8s objects

app/deploy/k8s/helpers.py

@@ -55,3 +55,50 @@ def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
 def log_stream_from_string(s: str):
     # Note response has to be UTF-8 encoded because the caller expects to decode it
     yield ("ignore", s.encode())
+
+
+def named_volumes_from_pod_files(parsed_pod_files):
+    # Parse the compose files looking for named volumes
+    named_volumes = []
+    for pod in parsed_pod_files:
+        parsed_pod_file = parsed_pod_files[pod]
+        if "volumes" in parsed_pod_file:
+            volumes = parsed_pod_file["volumes"]
+            for volume in volumes.keys():
+                # Volume definition looks like:
+                # 'laconicd-data': None
+                named_volumes.append(volume)
+    return named_volumes
+
+
+def volume_mounts_for_service(parsed_pod_files, service):
+    result = []
+    # Find the service
+    for pod in parsed_pod_files:
+        parsed_pod_file = parsed_pod_files[pod]
+        if "services" in parsed_pod_file:
+            services = parsed_pod_file["services"]
+            for service_name in services:
+                if service_name == service:
+                    service_obj = services[service_name]
+                    if "volumes" in service_obj:
+                        volumes = service_obj["volumes"]
+                        for mount_string in volumes:
+                            # Looks like: test-data:/data
+                            (volume_name, mount_path) = mount_string.split(":")
+                            volume_device = client.V1VolumeMount(mount_path=mount_path, name=volume_name)
+                            result.append(volume_device)
+    return result
+
+
+def volumes_for_pod_files(parsed_pod_files):
+    result = []
+    for pod in parsed_pod_files:
+        parsed_pod_file = parsed_pod_files[pod]
+        if "volumes" in parsed_pod_file:
+            volumes = parsed_pod_file["volumes"]
+            for volume_name in volumes.keys():
+                claim = client.V1PersistentVolumeClaimVolumeSource(claim_name=volume_name)
+                volume = client.V1Volume(name=volume_name, persistent_volume_claim=claim)
+                result.append(volume)
+    return result

tests/fixturenet-laconicd/run-test.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+echo "$(date +"%Y-%m-%d %T"): Running stack-orchestrator Laconicd fixturenet test"
+env
+cat /etc/hosts
+# Bit of a hack, test the most recent package
+TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+# Set a new unique repo dir
+export CERC_REPO_BASE_DIR=$(mktemp -d $(pwd)/stack-orchestrator-fixturenet-laconicd-test.XXXXXXXXXX)
+echo "$(date +"%Y-%m-%d %T"): Testing this package: $TEST_TARGET_SO"
+echo "$(date +"%Y-%m-%d %T"): Test version command"
+reported_version_string=$( $TEST_TARGET_SO version )
+echo "$(date +"%Y-%m-%d %T"): Version reported is: ${reported_version_string}"
+
+echo "$(date +"%Y-%m-%d %T"): Cloning laconicd repositories into: $CERC_REPO_BASE_DIR"
+$TEST_TARGET_SO --stack fixturenet-laconicd setup-repositories
+
+echo "$(date +"%Y-%m-%d %T"): Building containers"
+$TEST_TARGET_SO --stack fixturenet-laconicd build-containers
+echo "$(date +"%Y-%m-%d %T"): Starting stack"
+$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd up
+echo "$(date +"%Y-%m-%d %T"): Stack started"
+# Verify that the fixturenet is up and running
+$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd ps
+
+timeout=900 # 15 minutes
+echo "$(date +"%Y-%m-%d %T"): Getting initial block number. Timeout set to $timeout seconds"
+start_time=$(date +%s)
+elapsed_time=0
+initial_block_number=null
+while [ "$initial_block_number" == "null" ]  && [ $elapsed_time -lt $timeout ]; do
+  sleep 10
+  echo "$(date +"%Y-%m-%d %T"): Waiting for initial block..."
+  initial_block_number=$(docker exec laconicd-laconicd-1 /usr/bin/laconicd status | jq -r .SyncInfo.latest_block_height)
+  current_time=$(date +%s)
+  elapsed_time=$((current_time - start_time))
+done
+
+subsequent_block_number=$initial_block_number
+
+# if initial block was 0 after timeout, assume chain did not start successfully and skip finding subsequent block
+if [[ $initial_block_number != "null" ]]; then
+  timeout=300
+  echo "$(date +"%Y-%m-%d %T"): Getting subsequent block number. Timeout set to $timeout seconds"
+  start_time=$(date +%s)
+  elapsed_time=0
+  # wait for 5 blocks or timeout
+  while [ "$subsequent_block_number" -le $((initial_block_number + 5)) ]  && [ $elapsed_time -lt $timeout ]; do
+    sleep 10
+    echo "$(date +"%Y-%m-%d %T"): Waiting for five blocks or $timeout seconds..."
+    subsequent_block_number=$(docker exec laconicd-laconicd-1 /usr/bin/laconicd status | jq -r .SyncInfo.latest_block_height)
+    current_time=$(date +%s)
+    elapsed_time=$((current_time - start_time))
+  done
+fi
+
+# will return 0 if either of the above loops timed out
+block_number_difference=$((subsequent_block_number - initial_block_number))
+
+echo "$(date +"%Y-%m-%d %T"): Results of block height queries:"
+echo "Initial block height: $initial_block_number"
+echo "Subsequent block height: $subsequent_block_number"
+
+# Block height difference should be between 1 and some small number
+if [[ $block_number_difference -gt 1 && $block_number_difference -lt 100 ]]; then
+  echo "Test passed"
+  test_result=0
+else
+  echo "Test failed: block numbers were ${initial_block_number} and ${subsequent_block_number}"
+  echo "Logs from stack:"
+  $TEST_TARGET_SO --stack fixturenet-laconicd deploy logs
+  test_result=1
+fi
+
+$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd down --delete-volumes
+echo "$(date +"%Y-%m-%d %T"): Removing cloned repositories"
+rm -rf $CERC_REPO_BASE_DIR
+echo "$(date +"%Y-%m-%d %T"): Test finished"
+exit $test_result