Fix bad test in tag check. (#738)

Reviewed-on: cerc-io/stack-orchestrator#738
Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com>
Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>

requirements.txt

@@ -10,3 +10,4 @@ pydantic==1.10.9
 tomli==2.0.1
 validators==0.22.0
 kubernetes>=28.1.0
+humanfriendly>=10.0

stack_orchestrator/data/compose/docker-compose-mars.yml

@@ -0,0 +1,20 @@
+version: "3.2"
+
+services:
+  mars:
+    image: cerc/mars:local
+    restart: always
+    ports:
+      - "3000:3000"
+    environment:
+      - URL_OSMOSIS_GQL=https://osmosis-node.marsprotocol.io/GGSFGSFGFG34/osmosis-hive-front/graphql
+      - URL_OSMOSIS_REST=https://lcd-osmosis.blockapsis.com
+      - URL_OSMOSIS_RPC=https://rpc-osmosis.blockapsis.com
+      - URL_NEUTRON_GQL=https://neutron.rpc.p2p.world/qgrnU6PsQZA8F9S5Fb8Fn3tV3kXmMBl2M9bcc9jWLjQy8p/hive/graphql
+      - URL_NEUTRON_REST=https://rest-kralum.neutron-1.neutron.org
+      - URL_NEUTRON_RPC=https://rpc-kralum.neutron-1.neutron.org
+      - URL_NEUTRON_TEST_GQL=https://testnet-neutron-gql.marsprotocol.io/graphql
+      - URL_NEUTRON_TEST_REST=https://rest-palvus.pion-1.ntrn.tech
+      - URL_NEUTRON_TEST_RPC=https://rpc-palvus.pion-1.ntrn.tech
+      - WALLET_CONNECT_ID=0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x
+

stack_orchestrator/data/compose/docker-compose-ping-pub.yml

@@ -0,0 +1,8 @@
+version: "3.2"
+
+services:
+  ping-pub:
+    image: cerc/ping-pub:local
+    restart: always
+    ports:
+      - "5173:5173"

stack_orchestrator/data/compose/docker-compose-test.yml

@@ -5,6 +5,7 @@ services:
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_TEST_PARAM_1: ${CERC_TEST_PARAM_1:-FAILED}
+      CERC_TEST_PARAM_2: "CERC_TEST_PARAM_2_VALUE"
     volumes:
       - test-data:/data
       - test-config:/config:ro

stack_orchestrator/data/container-build/cerc-mars/build.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build the mars image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+docker build -t cerc/mars:local -f ${CERC_REPO_BASE_DIR}/mars-interface/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/mars-interface

stack_orchestrator/data/container-build/cerc-ping-pub/build.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Build the ping pub image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+docker build -t cerc/ping-pub:local ${build_command_args} -f $CERC_REPO_BASE_DIR/explorer/Dockerfile $CERC_REPO_BASE_DIR/explorer

stack_orchestrator/data/container-build/cerc-test-container/run.sh

@@ -17,6 +17,9 @@ fi
 if [ -n "$CERC_TEST_PARAM_1" ]; then
   echo "Test-param-1: ${CERC_TEST_PARAM_1}"
 fi
+if [ -n "$CERC_TEST_PARAM_2" ]; then
+  echo "Test-param-2: ${CERC_TEST_PARAM_2}"
+fi
 
 if [ -d "/config" ]; then
   echo "/config: EXISTS"

stack_orchestrator/data/stacks/mars/README.md

@@ -0,0 +1,16 @@
+# mars
+
+On a fresh Digital Ocean droplet with Ubuntu:
+
+```
+git clone https://github.com/cerc-io/stack-orchestrator
+cd stack-orchestrator
+./scripts/quick-install-linux.sh
+```
+Read and follow the instructions output from the above output to complete installation, then:
+
+```
+laconic-so --stack mars setup-repositories
+laconic-so --stack mars build-containers
+laconic-so --stack mars deploy up
+```

stack_orchestrator/data/stacks/mars/stack.yml

@@ -0,0 +1,8 @@
+version: "0.1"
+name: mars
+repos:
+  - github.com/cerc-io/mars-interface
+containers:
+  - cerc/mars
+pods:
+  - mars

stack_orchestrator/data/stacks/ping-pub/README.md

@@ -0,0 +1,10 @@
+# ping-pub
+Experimental block explorer for laconic
+
+```
+laconic-so --stack ping-pub setup-repositories
+laconic-so --stack ping-pub build-containers
+laconic-so --stack ping-pub deploy init --output ping-pub-spec.yml --map-ports-to-host localhost-same
+laconic-so --stack ping-pub deploy create --spec-file ping-pub-spec.yml --deployment-dir pp-deployment
+laconic-so deployment --dir pp-deployment start
+```

stack_orchestrator/data/stacks/ping-pub/stack.yml

@@ -0,0 +1,9 @@
+version: "0.1"
+name: ping-pub
+repos:
+  # fork, but only for config & Dockerfile reasonsb
+  - github.com/LaconicNetwork/explorer@laconic
+containers:
+  - cerc/ping-pub
+pods:
+  - ping-pub

stack_orchestrator/deploy/k8s/cluster_info.py

@@ -22,12 +22,41 @@ from stack_orchestrator.opts import opts
 from stack_orchestrator.util import env_var_map_from_file
 from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files
 from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path
-from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map
+from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map, envs_from_compose_file, merge_envs
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment
 from stack_orchestrator.deploy.deploy_types import DeployEnvVars
-from stack_orchestrator.deploy.spec import Spec
+from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits
 from stack_orchestrator.deploy.images import remote_tag_for_image
 
+DEFAULT_VOLUME_RESOURCES = Resources({
+    "reservations": {"storage": "2Gi"}
+})
+
+DEFAULT_CONTAINER_RESOURCES = Resources({
+    "reservations": {"cpus": "0.1", "memory": "200M"},
+    "limits": {"cpus": "1.0", "memory": "2000M"},
+})
+
+
+def to_k8s_resource_requirements(resources: Resources) -> client.V1ResourceRequirements:
+    def to_dict(limits: ResourceLimits):
+        if not limits:
+            return None
+
+        ret = {}
+        if limits.cpus:
+            ret["cpu"] = str(limits.cpus)
+        if limits.memory:
+            ret["memory"] = f"{int(limits.memory / (1000 * 1000))}M"
+        if limits.storage:
+            ret["storage"] = f"{int(limits.storage / (1000 * 1000))}M"
+        return ret
+
+    return client.V1ResourceRequirements(
+        requests=to_dict(resources.reservations),
+        limits=to_dict(resources.limits)
+    )
+
 
 class ClusterInfo:
     parsed_pod_yaml_map: Any
@@ -135,9 +164,13 @@ class ClusterInfo:
         result = []
         spec_volumes = self.spec.get_volumes()
         named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        resources = self.spec.get_volume_resources()
+        if not resources:
+            resources = DEFAULT_VOLUME_RESOURCES
         if opts.o.debug:
             print(f"Spec Volumes: {spec_volumes}")
             print(f"Named Volumes: {named_volumes}")
+            print(f"Resources: {resources}")
         for volume_name in spec_volumes:
             if volume_name not in named_volumes:
                 if opts.o.debug:
@@ -146,9 +179,7 @@ class ClusterInfo:
             spec = client.V1PersistentVolumeClaimSpec(
                 access_modes=["ReadWriteOnce"],
                 storage_class_name="manual",
-                resources=client.V1ResourceRequirements(
-                    requests={"storage": "2Gi"}
-                ),
+                resources=to_k8s_resource_requirements(resources),
                 volume_name=f"{self.app_name}-{volume_name}"
             )
             pvc = client.V1PersistentVolumeClaim(
@@ -192,6 +223,9 @@ class ClusterInfo:
         result = []
         spec_volumes = self.spec.get_volumes()
         named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        resources = self.spec.get_volume_resources()
+        if not resources:
+            resources = DEFAULT_VOLUME_RESOURCES
         for volume_name in spec_volumes:
             if volume_name not in named_volumes:
                 if opts.o.debug:
@@ -200,7 +234,7 @@ class ClusterInfo:
             spec = client.V1PersistentVolumeSpec(
                 storage_class_name="manual",
                 access_modes=["ReadWriteOnce"],
-                capacity={"storage": "2Gi"},
+                capacity=to_k8s_resource_requirements(resources).requests,
                 host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name))
             )
             pv = client.V1PersistentVolume(
@@ -214,6 +248,9 @@ class ClusterInfo:
     # TODO: put things like image pull policy into an object-scope struct
     def get_deployment(self, image_pull_policy: str = None):
         containers = []
+        resources = self.spec.get_container_resources()
+        if not resources:
+            resources = DEFAULT_CONTAINER_RESOURCES
         for pod_name in self.parsed_pod_yaml_map:
             pod = self.parsed_pod_yaml_map[pod_name]
             services = pod["services"]
@@ -226,6 +263,13 @@ class ClusterInfo:
                     if opts.o.debug:
                         print(f"image: {image}")
                         print(f"service port: {port}")
+                merged_envs = merge_envs(
+                    envs_from_compose_file(
+                        service_info["environment"]), self.environment_variables.map
+                        ) if "environment" in service_info else self.environment_variables.map
+                envs = envs_from_environment_variables_map(merged_envs)
+                if opts.o.debug:
+                    print(f"Merged envs: {envs}")
                 # Re-write the image tag for remote deployment
                 image_to_use = remote_tag_for_image(
                     image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
@@ -234,13 +278,10 @@ class ClusterInfo:
                     name=container_name,
                     image=image_to_use,
                     image_pull_policy=image_pull_policy,
-                    env=envs_from_environment_variables_map(self.environment_variables.map),
+                    env=envs,
                     ports=[client.V1ContainerPort(container_port=port)],
                     volume_mounts=volume_mounts,
-                    resources=client.V1ResourceRequirements(
-                        requests={"cpu": "100m", "memory": "200Mi"},
-                        limits={"cpu": "1000m", "memory": "2000Mi"},
-                    ),
+                    resources=to_k8s_resource_requirements(resources),
                 )
                 containers.append(container)
         volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec, self.app_name)

stack_orchestrator/deploy/k8s/helpers.py

@@ -17,6 +17,7 @@ from kubernetes import client
 import os
 from pathlib import Path
 import subprocess
+import re
 from typing import Set, Mapping, List
 
 from stack_orchestrator.opts import opts
@@ -214,6 +215,33 @@ def _generate_kind_port_mappings(parsed_pod_files):
     )
 
 
+# Note: this makes any duplicate definition in b overwrite a
+def merge_envs(a: Mapping[str, str], b: Mapping[str, str]) -> Mapping[str, str]:
+    result = {**a, **b}
+    return result
+
+
+def _expand_shell_vars(raw_val: str) -> str:
+    # could be: <string> or ${<env-var-name>} or ${<env-var-name>:-<default-value>}
+    # TODO: implement support for variable substitution and default values
+    # if raw_val is like ${<something>} print a warning and substitute an empty string
+    # otherwise return raw_val
+    match = re.search(r"^\$\{(.*)\}$", raw_val)
+    if match:
+        print(f"WARNING: found unimplemented environment variable substitution: {raw_val}")
+    else:
+        return raw_val
+
+
+# TODO: handle the case where the same env var is defined in multiple places
+def envs_from_compose_file(compose_file_envs: Mapping[str, str]) -> Mapping[str, str]:
+    result = {}
+    for env_var, env_val in compose_file_envs.items():
+        expanded_env_val = _expand_shell_vars(env_val)
+        result.update({env_var: expanded_env_val})
+    return result
+
+
 def envs_from_environment_variables_map(map: Mapping[str, str]) -> List[client.V1EnvVar]:
     result = []
     for env_var, env_val in map.items():

stack_orchestrator/deploy/spec.py

@@ -13,12 +13,60 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from pathlib import Path
 import typing
+import humanfriendly
+
+from pathlib import Path
+
 from stack_orchestrator.util import get_yaml
 from stack_orchestrator import constants
 
 
+class ResourceLimits:
+    cpus: float = None
+    memory: int = None
+    storage: int = None
+
+    def __init__(self, obj={}):
+        if "cpus" in obj:
+            self.cpus = float(obj["cpus"])
+        if "memory" in obj:
+            self.memory = humanfriendly.parse_size(obj["memory"])
+        if "storage" in obj:
+            self.storage = humanfriendly.parse_size(obj["storage"])
+
+    def __len__(self):
+        return len(self.__dict__)
+
+    def __iter__(self):
+        for k in self.__dict__:
+            yield k, self.__dict__[k]
+
+    def __repr__(self):
+        return str(self.__dict__)
+
+
+class Resources:
+    limits: ResourceLimits = None
+    reservations: ResourceLimits = None
+
+    def __init__(self, obj={}):
+        if "reservations" in obj:
+            self.reservations = ResourceLimits(obj["reservations"])
+        if "limits" in obj:
+            self.limits = ResourceLimits(obj["limits"])
+
+    def __len__(self):
+        return len(self.__dict__)
+
+    def __iter__(self):
+        for k in self.__dict__:
+            yield k, self.__dict__[k]
+
+    def __repr__(self):
+        return str(self.__dict__)
+
+
 class Spec:
 
     obj: typing.Any
@@ -47,6 +95,12 @@ class Spec:
                 if self.obj and "configmaps" in self.obj
                 else {})
 
+    def get_container_resources(self):
+        return Resources(self.obj.get("resources", {}).get("containers", {}))
+
+    def get_volume_resources(self):
+        return Resources(self.obj.get("resources", {}).get("volumes", {}))
+
     def get_http_proxy(self):
         return (self.obj[constants.network_key][constants.http_proxy_key]
                 if self.obj and constants.network_key in self.obj

stack_orchestrator/deploy/webapp/util.py

@@ -355,11 +355,13 @@ def generate_hostname_for_app(app):
 
 def skip_by_tag(r, include_tags, exclude_tags):
     for tag in exclude_tags:
-        if tag and r.attributes.tags and tag in r.attributes.tags:
+        if r.attributes.tags and tag in r.attributes.tags:
             return True
 
-    for tag in include_tags:
-        if tag and (not r.attributes.tags or tag not in r.attributes.tags):
-            return True
+    if include_tags:
+        for tag in include_tags:
+            if r.attributes.tags and tag in r.attributes.tags:
+                return False
+        return True
 
     return False

tests/deploy/run-deploy-test.sh

@@ -6,6 +6,12 @@ fi
 # Dump environment variables for debugging
 echo "Environment variables:"
 env
+
+delete_cluster_exit () {
+    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+    exit 1
+}
+
 # Test basic stack-orchestrator deploy
 echo "Running stack-orchestrator deploy test"
 # Bit of a hack, test the most recent package
@@ -106,6 +112,10 @@ if [ ! "$create_file_content" == "create-command-output-data"  ]; then
     echo "deploy create test: FAILED"
     exit 1
 fi
+
+# Add a config file to be picked up by the ConfigMap before starting.
+echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/data/test-config/test_config
+
 echo "deploy create output file test: passed"
 # Try to start the deployment
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
@@ -124,6 +134,37 @@ else
     echo "deployment config test: FAILED"
     exit 1
 fi
+# Check the config variable CERC_TEST_PARAM_2 was passed correctly from the compose file
+if [[ "$log_output_3" == *"Test-param-2: CERC_TEST_PARAM_2_VALUE"* ]]; then
+    echo "deployment compose config test: passed"
+else
+    echo "deployment compose config test: FAILED"
+    exit 1
+fi
+
+# Check that the ConfigMap is mounted and contains the expected content.
+log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then
+    echo "deployment ConfigMap test: passed"
+else
+    echo "deployment ConfigMap test: FAILED"
+    delete_cluster_exit
+fi
+
+# Stop then start again and check the volume was preserved
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
+# Sleep a bit just in case
+# sleep for longer to check if that's why the subsequent create cluster fails
+sleep 20
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_5" == *"Filesystem is old"* ]]; then
+    echo "Retain volumes test: passed"
+else
+    echo "Retain volumes test: FAILED"
+    delete_cluster_exit
+fi
+
 # Stop and clean up
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
 echo "Test passed"

tests/k8s-deploy/run-deploy-test.sh

@@ -114,6 +114,7 @@ else
     echo "deployment logs test: FAILED"
     delete_cluster_exit
 fi
+
 # Check the config variable CERC_TEST_PARAM_1 was passed correctly
 if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then
     echo "deployment config test: passed"
@@ -122,6 +123,14 @@ else
     delete_cluster_exit
 fi
 
+# Check the config variable CERC_TEST_PARAM_2 was passed correctly from the compose file
+if [[ "$log_output_3" == *"Test-param-2: CERC_TEST_PARAM_2_VALUE"* ]]; then
+    echo "deployment compose config test: passed"
+else
+    echo "deployment compose config test: FAILED"
+    exit 1
+fi
+
 # Check that the ConfigMap is mounted and contains the expected content.
 log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
 if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then