add newline

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -0,0 +1,55 @@
+name: Fixturenet-Laconicd-Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/fixturenet-laconicd-test'
+
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
+
+jobs:
+  test:
+    name: "Run an Laconicd fixturenet test"
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Update'
+        run: apt-get update
+      - name: 'Setup jq'
+        run: apt-get install jq -y
+      - name: 'Check jq'
+        run: |
+          which jq
+          jq --version
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: Start dockerd # Also needed until we can incorporate into the executor
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
+      - name: "Run fixturenet-laconicd tests"
+        run: ./tests/fixturenet-laconicd/run-test.sh

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -0,0 +1,2 @@
+Change this file to trigger running the fixturenet-laconicd-test CI job
+

.github/workflows/fixturenet-laconicd.yml

@@ -0,0 +1,36 @@
+name: Fixturenet-Laconicd Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.github/workflows/triggers/fixturenet-laconicd-test'
+
+jobs:
+  test:
+    name: "Run fixturenet-laconicd test suite"
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Setup jq'
+        run: apt-get install jq -y
+      - name: 'Check jq'
+        run: |
+          which jq
+          jq --version 
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Run fixturenet-laconicd tests"
+        run: ./tests/fixturenet-laconicd/run-test.sh

.github/workflows/triggers/fixturenet-laconicd-test

@@ -0,0 +1,3 @@
+Change this file to trigger running the fixturenet-laconicd-test CI job
+
+trigger

app/deploy/k8s/cluster_info.py

@@ -1,84 +0,0 @@
-# Copyright © 2023 Vulcanize
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
-
-from kubernetes import client
-from typing import Any, List, Set
-
-from app.opts import opts
-from app.util import get_yaml
-
-
-class ClusterInfo:
-    parsed_pod_yaml_map: Any = {}
-    image_set: Set[str] = set()
-    app_name: str = "test-app"
-    deployment_name: str = "test-deployment"
-
-    def __init__(self) -> None:
-        pass
-
-    def int_from_pod_files(self, pod_files: List[str]):
-        for pod_file in pod_files:
-            with open(pod_file, "r") as pod_file_descriptor:
-                parsed_pod_file = get_yaml().load(pod_file_descriptor)
-                self.parsed_pod_yaml_map[pod_file] = parsed_pod_file
-        if opts.o.debug:
-            print(f"parsed_pod_yaml_map: {self.parsed_pod_yaml_map}")
-        # Find the set of images in the pods
-        for pod_name in self.parsed_pod_yaml_map:
-            pod = self.parsed_pod_yaml_map[pod_name]
-            services = pod["services"]
-            for service_name in services:
-                service_info = services[service_name]
-                image = service_info["image"]
-                self.image_set.add(image)
-        if opts.o.debug:
-            print(f"image_set: {self.image_set}")
-
-    def get_deployment(self):
-        containers = []
-        for pod_name in self.parsed_pod_yaml_map:
-            pod = self.parsed_pod_yaml_map[pod_name]
-            services = pod["services"]
-            for service_name in services:
-                container_name = service_name
-                service_info = services[service_name]
-                image = service_info["image"]
-                container = client.V1Container(
-                    name=container_name,
-                    image=image,
-                    ports=[client.V1ContainerPort(container_port=80)],
-                    resources=client.V1ResourceRequirements(
-                        requests={"cpu": "100m", "memory": "200Mi"},
-                        limits={"cpu": "500m", "memory": "500Mi"},
-                    ),
-                )
-                containers.append(container)
-        template = client.V1PodTemplateSpec(
-            metadata=client.V1ObjectMeta(labels={"app": self.app_name}),
-            spec=client.V1PodSpec(containers=containers),
-        )
-        spec = client.V1DeploymentSpec(
-            replicas=1, template=template, selector={
-                "matchLabels":
-                {"app": self.app_name}})
-
-        deployment = client.V1Deployment(
-            api_version="apps/v1",
-            kind="Deployment",
-            metadata=client.V1ObjectMeta(name=self.deployment_name),
-            spec=spec,
-        )
-        return deployment

app/deploy/k8s/deploy_k8s.py

@@ -14,86 +14,33 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from kubernetes import client, config
-
 from app.deploy.deployer import Deployer
-from app.deploy.k8s.helpers import create_cluster, destroy_cluster, load_images_into_kind
-from app.deploy.k8s.helpers import pods_in_deployment, log_stream_from_string
-from app.deploy.k8s.cluster_info import ClusterInfo
-from app.opts import opts
 
 
 class K8sDeployer(Deployer):
     name: str = "k8s"
-    core_api: client.CoreV1Api
-    apps_api: client.AppsV1Api
-    kind_cluster_name: str
-    cluster_info : ClusterInfo
 
     def __init__(self, compose_files, compose_project_name, compose_env_file) -> None:
-        if (opts.o.debug):
-            print(f"Compose files: {compose_files}")
-            print(f"Project name: {compose_project_name}")
-            print(f"Env file: {compose_env_file}")
-        self.kind_cluster_name = compose_project_name
-        self.cluster_info = ClusterInfo()
-        self.cluster_info.int_from_pod_files(compose_files)
-
-    def connect_api(self):
-        config.load_kube_config(context=f"kind-{self.kind_cluster_name}")
-        self.core_api = client.CoreV1Api()
-        self.apps_api = client.AppsV1Api()
+        config.load_kube_config()
+        self.client = client.CoreV1Api()
 
     def up(self, detach, services):
-        # Create the kind cluster
-        create_cluster(self.kind_cluster_name)
-        self.connect_api()
-        # Ensure the referenced containers are copied into kind
-        load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set)
-        # Process compose files into a Deployment
-        deployment = self.cluster_info.get_deployment()
-        # Create the k8s objects
-        resp = self.apps_api.create_namespaced_deployment(
-            body=deployment, namespace="default"
-        )
-
-        if opts.o.debug:
-            print("Deployment created.\n")
-            print(f"{resp.metadata.namespace} {resp.metadata.name} \
-                  {resp.metadata.generation} {resp.spec.template.spec.containers[0].image}")
+        pass
 
     def down(self, timeout, volumes):
-        # Delete the k8s objects
-        # Destroy the kind cluster
-        destroy_cluster(self.kind_cluster_name)
+        pass
 
     def ps(self):
-        self.connect_api()
-        # Call whatever API we need to get the running container list
-        ret = self.core_api.list_pod_for_all_namespaces(watch=False)
-        if ret.items:
-            for i in ret.items:
-                print("%s\t%s\t%s" % (i.status.pod_ip, i.metadata.namespace, i.metadata.name))
-        ret = self.core_api.list_node(pretty=True, watch=False)
-        return []
+        pass
 
     def port(self, service, private_port):
-        # Since we handle the port mapping, need to figure out where this comes from
-        # Also look into whether it makes sense to get ports for k8s
         pass
 
     def execute(self, service_name, command, envs):
-        # Call the API to execute a command in a running container
         pass
 
     def logs(self, services, tail, follow, stream):
-        self.connect_api()
-        pods = pods_in_deployment(self.core_api, "test-deployment")
-        if len(pods) > 1:
-            print("Warning: more than one pod in the deployment")
-        k8s_pod_name = pods[0]
-        log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
-        return log_stream_from_string(log_data)
+        pass
 
     def run(self, image, command, user, volumes, entrypoint=None):
-        # We need to figure out how to do this -- check why we're being called first
         pass

app/deploy/k8s/helpers.py

@@ -1,57 +0,0 @@
-# Copyright © 2023 Vulcanize
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
-
-from kubernetes import client
-import subprocess
-from typing import Set
-
-from app.opts import opts
-
-
-def _run_command(command: str):
-    if opts.o.debug:
-        print(f"Running: {command}")
-    result = subprocess.run(command, shell=True)
-    if opts.o.debug:
-        print(f"Result: {result}")
-
-
-def create_cluster(name: str):
-    _run_command(f"kind create cluster --name {name}")
-
-
-def destroy_cluster(name: str):
-    _run_command(f"kind delete cluster --name {name}")
-
-
-def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]):
-    for image in image_set:
-        _run_command(f"kind load docker-image {image} --name {kind_cluster_name}")
-
-
-def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
-    pods = []
-    pod_response = core_api.list_namespaced_pod(namespace="default", label_selector="app=test-app")
-    if opts.o.debug:
-        print(f"pod_response: {pod_response}")
-    for pod_info in pod_response.items:
-        pod_name = pod_info.metadata.name
-        pods.append(pod_name)
-    return pods
-
-
-def log_stream_from_string(s: str):
-    # Note response has to be UTF-8 encoded because the caller expects to decode it
-    yield ("ignore", s.encode())

app/opts.py

@@ -1,20 +0,0 @@
-# Copyright © 2023 Vulcanize
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
-
-from app.command_types import CommandOptions
-
-
-class opts:
-    o: CommandOptions = None

cli.py

@@ -22,7 +22,6 @@ from app.build import build_npms
 from app.deploy import deploy
 from app import version
 from app.deploy import deployment
-from app import opts
 from app import update
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
@@ -40,9 +39,7 @@ CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 @click.pass_context
 def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error):
     """Laconic Stack Orchestrator"""
-    command_options = CommandOptions(stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error)
-    opts.opts.o = command_options
-    ctx.obj = command_options
+    ctx.obj = CommandOptions(stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error)
 
 
 cli.add_command(setup_repositories.command, "setup-repositories")

tests/fixturenet-laconicd/run-test.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+echo "$(date +"%Y-%m-%d %T"): Running stack-orchestrator Laconicd fixturenet test"
+env
+cat /etc/hosts
+# Bit of a hack, test the most recent package
+TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+# Set a new unique repo dir
+export CERC_REPO_BASE_DIR=$(mktemp -d $(pwd)/stack-orchestrator-fixturenet-laconicd-test.XXXXXXXXXX)
+echo "$(date +"%Y-%m-%d %T"): Testing this package: $TEST_TARGET_SO"
+echo "$(date +"%Y-%m-%d %T"): Test version command"
+reported_version_string=$( $TEST_TARGET_SO version )
+echo "$(date +"%Y-%m-%d %T"): Version reported is: ${reported_version_string}"
+
+echo "$(date +"%Y-%m-%d %T"): Cloning laconicd repositories into: $CERC_REPO_BASE_DIR"
+$TEST_TARGET_SO --stack fixturenet-laconicd setup-repositories
+
+echo "$(date +"%Y-%m-%d %T"): Building containers"
+$TEST_TARGET_SO --stack fixturenet-laconicd build-containers
+echo "$(date +"%Y-%m-%d %T"): Starting stack"
+$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd up
+echo "$(date +"%Y-%m-%d %T"): Stack started"
+# Verify that the fixturenet is up and running
+$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd ps
+
+timeout=900 # 15 minutes
+echo "$(date +"%Y-%m-%d %T"): Getting initial block number. Timeout set to $timeout seconds"
+start_time=$(date +%s)
+elapsed_time=0
+initial_block_number=null
+while [ "$initial_block_number" == "null" ]  && [ $elapsed_time -lt $timeout ]; do
+  sleep 10
+  echo "$(date +"%Y-%m-%d %T"): Waiting for initial block..."
+  initial_block_number=$(docker exec laconicd-laconicd-1 /usr/bin/laconicd status | jq -r .SyncInfo.latest_block_height)
+  current_time=$(date +%s)
+  elapsed_time=$((current_time - start_time))
+done
+
+subsequent_block_number=$initial_block_number
+
+# if initial block was 0 after timeout, assume chain did not start successfully and skip finding subsequent block
+if [[ $initial_block_number != "null" ]]; then
+  timeout=300
+  echo "$(date +"%Y-%m-%d %T"): Getting subsequent block number. Timeout set to $timeout seconds"
+  start_time=$(date +%s)
+  elapsed_time=0
+  # wait for 5 blocks or timeout
+  while [ "$subsequent_block_number" -le $((initial_block_number + 5)) ]  && [ $elapsed_time -lt $timeout ]; do
+    sleep 10
+    echo "$(date +"%Y-%m-%d %T"): Waiting for five blocks or $timeout seconds..."
+    subsequent_block_number=$(docker exec laconicd-laconicd-1 /usr/bin/laconicd status | jq -r .SyncInfo.latest_block_height)
+    current_time=$(date +%s)
+    elapsed_time=$((current_time - start_time))
+  done
+fi
+
+# will return 0 if either of the above loops timed out
+block_number_difference=$((subsequent_block_number - initial_block_number))
+
+echo "$(date +"%Y-%m-%d %T"): Results of block height queries:"
+echo "Initial block height: $initial_block_number"
+echo "Subsequent block height: $subsequent_block_number"
+
+# Block height difference should be between 1 and some small number
+if [[ $block_number_difference -gt 1 && $block_number_difference -lt 100 ]]; then
+  echo "Test passed"
+  test_result=0
+else
+  echo "Test failed: block numbers were ${initial_block_number} and ${subsequent_block_number}"
+  echo "Logs from stack:"
+  $TEST_TARGET_SO --stack fixturenet-laconicd deploy logs
+  test_result=1
+fi
+
+$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd down --delete-volumes
+echo "$(date +"%Y-%m-%d %T"): Removing cloned repositories"
+rm -rf $CERC_REPO_BASE_DIR
+echo "$(date +"%Y-%m-%d %T"): Test finished"
+exit $test_result