Merge branch 'main' into ci-test

Change to trigger CI on Gitea

.gitea/workflows/test-container-registry.yml

@@ -1,54 +0,0 @@
-name: Container Registry Test
-
-on:
-  push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.gitea/workflows/triggers/test-container-registry'
-      - '.gitea/workflows/test-container-registry.yml'
-      - 'tests/container-registry/run-test.sh'
-  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
-    - cron: '6 19 * * *'
-
-jobs:
-  test:
-    name: "Run contaier registry hosting test on kind/k8s"
-    runs-on: ubuntu-22.04
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: "Check cgroups version"
-        run: mount | grep cgroup
-      - name: "Install kind"
-        run: ./tests/scripts/install-kind.sh
-      - name: "Install Kubectl"
-        run: ./tests/scripts/install-kubectl.sh
-      - name: "Install ed" # Only needed until we remove the need to edit the spec file
-        run: apt update && apt install -y ed
-      - name: "Run container registry deployment test"
-        run: |
-          source /opt/bash-utils/cgroup-helper.sh
-          join_cgroup
-          ./tests/container-registry/run-test.sh
-

.gitea/workflows/test-database.yml

@@ -48,5 +48,5 @@ jobs:
         run: |
           source /opt/bash-utils/cgroup-helper.sh
           join_cgroup
-          ./tests/database/run-test.sh
+          CERC_SCRIPT_DEBUG=1 ./tests/database/run-test.sh
 

.gitea/workflows/triggers/test-container-registry

@@ -1 +0,0 @@
-Change this file to trigger running the test-container-registry CI job

scripts/quick-deploy-test.sh

@@ -12,8 +12,8 @@ spec_file_name="${stack_name}-spec.yml"
 deployment_dir_name="${stack_name}-deployment"
 rm -f ${spec_file_name}
 rm -rf ${deployment_dir_name}
-laconic-so --stack ${stack_name} deploy --deploy-to compose init --output ${spec_file_name}
+laconic-so --stack ${stack_name} deploy --deploy-to k8s-kind init --output ${spec_file_name}
-laconic-so --stack ${stack_name} deploy --deploy-to compose create --deployment-dir ${deployment_dir_name} --spec-file ${spec_file_name}
+laconic-so --stack ${stack_name} deploy --deploy-to k8s-kind create --deployment-dir ${deployment_dir_name} --spec-file ${spec_file_name}
 #laconic-so deployment --dir ${deployment_dir_name} start
 #laconic-so deployment --dir ${deployment_dir_name} ps
 #laconic-so deployment --dir ${deployment_dir_name} stop

stack_orchestrator/build/build_containers.py

@@ -27,7 +27,7 @@ import subprocess
 import click
 import importlib.resources
 from pathlib import Path
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, warn_exit
+from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external
 from stack_orchestrator.base import get_npm_registry_url
 
 # TODO: find a place for this
@@ -164,8 +164,6 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
     containers_in_scope = []
     if stack:
         stack_config = get_parsed_stack_config(stack)
-        if "containers" not in stack_config or stack_config["containers"] is None:
-            warn_exit(f"stack {stack} does not define any containers")
         containers_in_scope = stack_config['containers']
     else:
         containers_in_scope = all_containers

stack_orchestrator/data/compose/docker-compose-container-registry.yml

@@ -1,13 +0,0 @@
-services:
-  registry:
-    image: registry:2.8
-    restart: always
-    environment:
-      REGISTRY_LOG_LEVEL: ${REGISTRY_LOG_LEVEL}
-    volumes:
-      - registry-data:/var/lib/registry
-    ports:
-      - "5000"
-
-volumes:
-  registry-data:

stack_orchestrator/data/compose/docker-compose-mars-v2.yml

@@ -1,12 +0,0 @@
-version: "3.2"
-
-services:
-  mars:
-    image: cerc/mars-v2:local
-    restart: always
-    ports:
-      - "3000:3000"
-    environment:
-      - URL_OSMOSIS_REST=https://lcd-osmosis.blockapsis.com
-      - URL_OSMOSIS_RPC=https://rpc-osmosis.blockapsis.com
-      - WALLET_CONNECT_ID=0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x

stack_orchestrator/data/container-build/cerc-mars-v2/build.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-# Build the mars-v2 image
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-docker build -t cerc/mars-v2:local -f ${CERC_REPO_BASE_DIR}/mars-v2-frontend/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/mars-v2-frontend

stack_orchestrator/data/stacks/container-registry/README.md

@@ -1,3 +0,0 @@
-# Container Registry Stack
-
-Host a container image registry

stack_orchestrator/data/stacks/container-registry/stack.yml

@@ -1,5 +0,0 @@
-version: "1.0"
-name: container-registry
-description: "Container registry stack"
-pods:
-  - container-registry

stack_orchestrator/data/stacks/mars-v2/README.md

@@ -1,16 +0,0 @@
-# mars
-
-On a fresh Digital Ocean droplet with Ubuntu:
-
-```
-git clone https://github.com/cerc-io/stack-orchestrator
-cd stack-orchestrator
-./scripts/quick-install-linux.sh
-```
-Read and follow the instructions output from the above output to complete installation, then:
-
-```
-laconic-so --stack mars-v2 setup-repositories
-laconic-so --stack mars-v2 build-containers
-laconic-so --stack mars-v2 deploy up
-```

stack_orchestrator/data/stacks/mars-v2/stack.yml

@@ -1,8 +0,0 @@
-version: "0.1"
-name: mars-v2
-repos:
-  - github.com/mars-protocol/mars-v2-frontend
-containers:
-  - cerc/mars-v2
-pods:
-  - mars-v2

tests/container-registry/run-test.sh

@@ -1,146 +0,0 @@
-#!/usr/bin/env bash
-set -e
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-    set -x
-    # Dump environment variables for debugging
-    echo "Environment variables:"
-    env
-fi
-
-stack="container-registry"
-
-# Helper functions: TODO move into a separate file
-wait_for_pods_started () {
-    for i in {1..50}
-    do
-        local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps )
-
-        if [[ "$ps_output" == *"Running containers:"* ]]; then
-            # if ready, return
-            return
-        else
-            # if not ready, wait
-            sleep 5
-        fi
-    done
-    # Timed out, error exit
-    echo "waiting for pods to start: FAILED"
-    delete_cluster_exit
-}
-
-wait_for_log_output () {
-    for i in {1..50}
-    do
-
-        local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-
-        if [[ ! -z "$log_output" ]]; then
-            # if ready, return
-            return
-        else
-            # if not ready, wait
-            sleep 5
-        fi
-    done
-    # Timed out, error exit
-    echo "waiting for pods log content: FAILED"
-    delete_cluster_exit
-}
-
-
-delete_cluster_exit () {
-    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
-    exit 1
-}
-
-# Note: eventually this test should be folded into ../deploy/
-# but keeping it separate for now for convenience
-TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
-# Set a non-default repo dir
-export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
-echo "Testing this package: $TEST_TARGET_SO"
-echo "Test version command"
-reported_version_string=$( $TEST_TARGET_SO version )
-echo "Version reported is: ${reported_version_string}"
-echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
-rm -rf $CERC_REPO_BASE_DIR
-mkdir -p $CERC_REPO_BASE_DIR
-$TEST_TARGET_SO --stack ${stack} setup-repositories
-$TEST_TARGET_SO --stack ${stack} build-containers
-# Test basic stack-orchestrator deploy to k8s
-test_deployment_dir=$CERC_REPO_BASE_DIR/${stack}-deployment-dir
-test_deployment_spec=$CERC_REPO_BASE_DIR/${stack}-deployment-spec.yml
-$TEST_TARGET_SO --stack ${stack} deploy --deploy-to k8s-kind init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED
-# Check the file now exists
-if [ ! -f "$test_deployment_spec" ]; then
-    echo "deploy init test: spec file not present"
-    echo "deploy init test: FAILED"
-    exit 1
-fi
-echo "deploy init test: passed"
-
-# Switch to a full path for bind mount.
-volume_name="registry-data"
-sed -i "s|^\(\s*${volume_name}:$\)$|\1 ${test_deployment_dir}/data/${volume_name}|" $test_deployment_spec
-
-# Add ingress config to the spec file
-ed $test_deployment_spec <<IngressSpec
-/network:/
-a
-  http-proxy:
-    - host-name: localhost
-      routes:
-        - path: /
-          proxy-to: registry:5000
-.
-w
-q
-IngressSpec
-
-$TEST_TARGET_SO --stack ${stack} deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
-# Check the deployment dir exists
-if [ ! -d "$test_deployment_dir" ]; then
-    echo "deploy create test: deployment directory not present"
-    echo "deploy create test: FAILED"
-    exit 1
-fi
-echo "deploy create test: passed"
-
-# Note: this isn't strictly necessary, except we end up trying to push the image into
-# the kind cluster then fails because it can't be found locally
-docker pull registry:2.8
-
-# Try to start the deployment
-$TEST_TARGET_SO deployment --dir $test_deployment_dir start
-wait_for_pods_started
-# Check logs command works
-wait_for_log_output
-sleep 1
-log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
-if [[ "$log_output_3" == *"listening on"* ]]; then
-    echo "deployment logs test: passed"
-else
-    echo "deployment logs test: FAILED"
-    echo $log_output_3
-    delete_cluster_exit
-fi
-
-# Check that we can use the registry
-# Note: since this pulls from the DockerCo registry without auth it's possible it'll run into rate limiting issues
-docker pull hello-world
-docker tag hello-world localhost:80/hello-world
-docker push localhost:80/hello-world
-# Then do a quick check that we actually pushed something there
-# See: https://stackoverflow.com/questions/31251356/how-to-get-a-list-of-images-on-docker-registry-v2
-registry_response=$(curl -s -X GET http://localhost:80/v2/_catalog)
-if [[ "$registry_response" == *"{\"repositories\":[\"hello-world\"]}"* ]]; then
-    echo "registry content test: passed"
-else
-    echo "registry content test: FAILED"
-    echo $registry_response
-    delete_cluster_exit
-fi
-
-# Stop and clean up
-$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
-echo "Test passed"