From 407571e18d25bfea164354957199964b7d25866f Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 15 Aug 2023 11:49:17 -0600 Subject: [PATCH 1/5] Add debian setup script (#501) * Adapt install script to Debian * Update docs for new scipt name * Add reference to dockerco install doc --- README.md | 2 +- docs/laconicd-fixturenet.md | 2 +- ...stall-ubuntu.sh => quick-install-linux.sh} | 39 +++++++++++++++++-- 3 files changed, 37 insertions(+), 6 deletions(-) rename scripts/{quick-install-ubuntu.sh => quick-install-linux.sh} (79%) diff --git a/README.md b/README.md index 4f490d8c..8dd6f041 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Stack Orchestrator allows building and deployment of a Laconic Stack on a single ## Install -**To get started quickly** on a fresh Ubuntu instance (e.g, Digital Ocean); [try this script](./scripts/quick-install-ubuntu.sh). **WARNING:** always review scripts prior to running them so that you know what is happening on your machine. +**To get started quickly** on a fresh Ubuntu instance (e.g, Digital Ocean); [try this script](./scripts/quick-install-linux.sh). **WARNING:** always review scripts prior to running them so that you know what is happening on your machine. For any other installation, follow along below and **adapt these instructions based on the specifics of your system.** diff --git a/docs/laconicd-fixturenet.md b/docs/laconicd-fixturenet.md index b2375b16..79a6336a 100644 --- a/docs/laconicd-fixturenet.md +++ b/docs/laconicd-fixturenet.md @@ -23,7 +23,7 @@ ssh root@IP 2. Get the install script, give it executable permissions, and run it: ``` -curl -o install.sh https://raw.githubusercontent.com/cerc-io/stack-orchestrator/main/scripts/quick-install-ubuntu.sh +curl -o install.sh https://raw.githubusercontent.com/cerc-io/stack-orchestrator/main/scripts/quick-install-linux.sh ``` ``` chmod +x install.sh diff --git a/scripts/quick-install-ubuntu.sh b/scripts/quick-install-linux.sh similarity index 79% rename from scripts/quick-install-ubuntu.sh rename to scripts/quick-install-linux.sh index e28c2052..4f4d044b 100755 --- a/scripts/quick-install-ubuntu.sh +++ b/scripts/quick-install-linux.sh @@ -28,12 +28,28 @@ fi # Check python3 is available # Check machine resources are sufficient +# Determine if we are on Debian or Ubuntu +linux_distro=$(lsb_release -a 2>/dev/null | grep "^Distributor ID:" | cut -f 2) +case $linux_distro in + Debian) + echo "Installing docker for Debian" + ;; + Ubuntu) + echo "Installing docker for Ubuntu" + ;; + *) + echo "ERROR: Detected unknown distribution $linux_distro, can't install docker" + exit 1 + ;; +esac + # dismiss the popups export DEBIAN_FRONTEND=noninteractive ## https://docs.docker.com/engine/install/ubuntu/ +## https://docs.docker.com/engine/install/debian/ ## https://superuser.com/questions/518859/ignore-packages-that-are-not-currently-installed-when-using-apt-get-remove1 -packages_to_remove="docker docker-engine docker.io containerd runc" +packages_to_remove="docker docker-engine docker.io containerd runc docker-compose docker-doc podman-docker" installed_packages_to_remove="" for package_to_remove in $(echo $packages_to_remove); do $(dpkg --info $package_to_remove &> /dev/null) @@ -65,10 +81,25 @@ sudo apt -y install curl sudo apt -y install ca-certificates gnupg # Add dockerco package repository -sudo mkdir -m 0755 -p /etc/apt/keyrings -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +# For reasons not obvious, the dockerco instructions for installation on +# Debian and Ubuntu are slightly different here +case $linux_distro in + Debian) + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + sudo chmod a+r /etc/apt/keyrings/docker.gpg + ;; + Ubuntu) + sudo mkdir -m 0755 -p /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + ;; + *) + echo "ERROR: Detected unknown distribution $linux_distro, can't install docker" + exit 1 + ;; +esac echo \ - "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/${linux_distro,,} \ "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null From d35bbbab9829cdf0a4bd35019b853ef8d7a7dfc6 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 17 Aug 2023 02:02:57 -0600 Subject: [PATCH 2/5] Support ChromeOS Debian VM (#503) --- scripts/quick-install-linux.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/quick-install-linux.sh b/scripts/quick-install-linux.sh index 4f4d044b..007102f6 100755 --- a/scripts/quick-install-linux.sh +++ b/scripts/quick-install-linux.sh @@ -30,6 +30,21 @@ fi # Determine if we are on Debian or Ubuntu linux_distro=$(lsb_release -a 2>/dev/null | grep "^Distributor ID:" | cut -f 2) +# Some systems don't have lsb_release installed (e.g. ChromeOS) and so we try to +# use /etc/os-release instead +if [[ -z "$linux_distro" ]]; then + if [[ -f "/etc/os-release" ]]; then + distro_name_string=$(grep "^NAME=" /etc/os-release | cut -d '=' -f 2) + if [[ $distro_name_string =~ Debian ]]; then + linux_distro="Debian" + elif [[ $distro_name_string =~ Ubuntu ]]; then + linux_distro="Ubuntu" + fi + else + echo "Failed to identify distro: /etc/os-release doesn't exist" + exit 1 + fi +fi case $linux_distro in Debian) echo "Installing docker for Debian" From 8a7622ffa902d64bd2546749acc6b34908c7df26 Mon Sep 17 00:00:00 2001 From: Zach Date: Thu, 17 Aug 2023 06:37:26 -0400 Subject: [PATCH 3/5] bump azimuth version (#502) --- app/data/stacks/azimuth/stack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/data/stacks/azimuth/stack.yml b/app/data/stacks/azimuth/stack.yml index bb7f2a88..47e0d058 100644 --- a/app/data/stacks/azimuth/stack.yml +++ b/app/data/stacks/azimuth/stack.yml @@ -1,7 +1,7 @@ version: "1.0" name: azimuth repos: - - github.com/cerc-io/azimuth-watcher-ts + - github.com/cerc-io/azimuth-watcher-ts@v0.1.1 containers: - cerc/watcher-azimuth pods: From 97dc45549d5074b394cd8d29c11a44b0730a5a08 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 17 Aug 2023 13:24:06 -0600 Subject: [PATCH 4/5] Update fixturenet eth test (#506) --- .../workflows/fixturenet-eth-plugeth-test.yml | 36 +++++++++++++++ .gitea/workflows/fixturenet-eth-test.yml | 10 +++++ .gitea/workflows/test.yml | 1 + .../container-build/cerc-lighthouse/build.sh | 2 +- tests/fixturenet-eth-plugeth/run-test.sh | 44 +++++++++++++++++++ tests/fixturenet-eth/run-test.sh | 29 +++++++----- 6 files changed, 111 insertions(+), 11 deletions(-) create mode 100644 .gitea/workflows/fixturenet-eth-plugeth-test.yml create mode 100755 tests/fixturenet-eth-plugeth/run-test.sh diff --git a/.gitea/workflows/fixturenet-eth-plugeth-test.yml b/.gitea/workflows/fixturenet-eth-plugeth-test.yml new file mode 100644 index 00000000..ced0823f --- /dev/null +++ b/.gitea/workflows/fixturenet-eth-plugeth-test.yml @@ -0,0 +1,36 @@ +name: Fixturenet-Eth-Plugeth-Test + +on: + push: + branches: 'ci-test' + +# Needed until we can incorporate docker startup into the executor container +env: + DOCKER_HOST: unix:///var/run/dind.sock + + +jobs: + test: + name: "Run an Ethereum plugeth fixturenet test" + runs-on: ubuntu-latest + steps: + - name: "Clone project repository" + uses: actions/checkout@v3 + - name: "Install Python" + uses: cerc-io/setup-python@v4 + with: + python-version: '3.8' + - name: "Print Python version" + run: python3 --version + - name: "Install shiv" + run: pip install shiv + - name: "Generate build version file" + run: ./scripts/create_build_tag_file.sh + - name: "Build local shiv package" + run: ./scripts/build_shiv_package.sh + - name: Start dockerd # Also needed until we can incorporate into the executor + run: | + dockerd -H $DOCKER_HOST --userland-proxy=false & + sleep 5 + - name: "Run fixturenet-eth tests" + run: ./tests/fixturenet-eth-plugeth/run-test.sh diff --git a/.gitea/workflows/fixturenet-eth-test.yml b/.gitea/workflows/fixturenet-eth-test.yml index 43572c92..5a0410f2 100644 --- a/.gitea/workflows/fixturenet-eth-test.yml +++ b/.gitea/workflows/fixturenet-eth-test.yml @@ -4,6 +4,11 @@ on: push: branches: 'ci-test' +# Needed until we can incorporate docker startup into the executor container +env: + DOCKER_HOST: unix:///var/run/dind.sock + + jobs: test: name: "Run an Ethereum fixturenet test" @@ -23,5 +28,10 @@ jobs: run: ./scripts/create_build_tag_file.sh - name: "Build local shiv package" run: ./scripts/build_shiv_package.sh + - name: Start dockerd # Also needed until we can incorporate into the executor + run: | + dockerd -H $DOCKER_HOST --userland-proxy=false & + sleep 5 - name: "Run fixturenet-eth tests" run: ./tests/fixturenet-eth/run-test.sh + diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 303b42f4..46c5f937 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -37,3 +37,4 @@ jobs: sleep 5 - name: "Run smoke tests" run: ./tests/smoke-test/run-smoke-test.sh + diff --git a/app/data/container-build/cerc-lighthouse/build.sh b/app/data/container-build/cerc-lighthouse/build.sh index 2e9cfe3c..cdc95612 100755 --- a/app/data/container-build/cerc-lighthouse/build.sh +++ b/app/data/container-build/cerc-lighthouse/build.sh @@ -6,4 +6,4 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh # See: https://stackoverflow.com/a/246128/1701505 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -docker build -t cerc/lighthouse:local ${build_command_args} ${SCRIPT_DIR} +docker build -t cerc/lighthouse:local ${build_command_args} --build-arg TAG_SUFFIX="" ${SCRIPT_DIR} diff --git a/tests/fixturenet-eth-plugeth/run-test.sh b/tests/fixturenet-eth-plugeth/run-test.sh new file mode 100755 index 00000000..1a961b56 --- /dev/null +++ b/tests/fixturenet-eth-plugeth/run-test.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +set -e +echo "Running stack-orchestrator Ethereum plugeth fixturenet test" +# Bit of a hack, test the most recent package +TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 ) +CERC_STACK_NAME=fixturenet-plugeth-tx +# Set a new unique repo dir +export CERC_REPO_BASE_DIR=$(mktemp -d stack-orchestrator-fixturenet-eth-test.XXXXXXXXXX) +echo "Testing this package: $TEST_TARGET_SO" +echo "Test version command" +reported_version_string=$( $TEST_TARGET_SO version ) +echo "Version reported is: ${reported_version_string}" +echo "Cloning repositories into: $CERC_REPO_BASE_DIR" +$TEST_TARGET_SO --stack $CERC_STACK_NAME setup-repositories +echo "Building containers" +$TEST_TARGET_SO --stack $CERC_STACK_NAME build-containers +echo "Images in registry:" +docker image ls +echo "Deploying the cluster" +$TEST_TARGET_SO --stack $CERC_STACK_NAME deploy up +# Verify that the fixturenet is up and running +$TEST_TARGET_SO --stack $CERC_STACK_NAME deploy ps +$TEST_TARGET_SO --stack $CERC_STACK_NAME deploy exec fixturenet-eth-bootnode-lighthouse /scripts/status-internal.sh +initial_block_number=$($TEST_TARGET_SO --stack fixturenet-plugeth-tx deploy exec foundry "cast block-number") +# Check that the block number increases some time later +sleep 12 +subsequent_block_number=$($TEST_TARGET_SO --stack $CERC_STACK_NAME deploy exec foundry "cast block-number") +block_number_difference=$((subsequent_block_number - initial_block_number)) +# Block height difference should be between 1 and some small number +if [[ $block_number_difference -gt 1 && $block_number_difference -lt 10 ]]; then + echo "Test passed" + test_result=0 +else + echo "Test failed: block numbers were ${initial_block_number} and ${subsequent_block_number}" + test_result=1 +fi +$TEST_TARGET_SO --stack $CERC_STACK_NAME deploy down +echo "Removing cloned repositories" +rm -rf $CERC_REPO_BASE_DIR +exit $test_result diff --git a/tests/fixturenet-eth/run-test.sh b/tests/fixturenet-eth/run-test.sh index a7e04030..b5654584 100755 --- a/tests/fixturenet-eth/run-test.sh +++ b/tests/fixturenet-eth/run-test.sh @@ -4,36 +4,45 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi -echo "Running stack-orchestrator Ethereum fixturenet test" +echo "$(date +"%Y-%m-%d %T"): Running stack-orchestrator Ethereum fixturenet test" # Bit of a hack, test the most recent package TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 ) # Set a new unique repo dir export CERC_REPO_BASE_DIR=$(mktemp -d stack-orchestrator-fixturenet-eth-test.XXXXXXXXXX) -echo "Testing this package: $TEST_TARGET_SO" -echo "Test version command" +echo "$(date +"%Y-%m-%d %T"): Testing this package: $TEST_TARGET_SO" +echo "$(date +"%Y-%m-%d %T"): Test version command" reported_version_string=$( $TEST_TARGET_SO version ) -echo "Version reported is: ${reported_version_string}" -echo "Cloning repositories into: $CERC_REPO_BASE_DIR" -$TEST_TARGET_SO --stack fixturenet-eth setup-repositories +echo "$(date +"%Y-%m-%d %T"): Version reported is: ${reported_version_string}" +echo "$(date +"%Y-%m-%d %T"): Cloning repositories into: $CERC_REPO_BASE_DIR" +$TEST_TARGET_SO --stack fixturenet-eth setup-repositories +echo "$(date +"%Y-%m-%d %T"): Building containers" $TEST_TARGET_SO --stack fixturenet-eth build-containers +echo "$(date +"%Y-%m-%d %T"): Starting stack" $TEST_TARGET_SO --stack fixturenet-eth deploy up +echo "$(date +"%Y-%m-%d %T"): Stack started" # Verify that the fixturenet is up and running $TEST_TARGET_SO --stack fixturenet-eth deploy ps -$TEST_TARGET_SO --stack fixturenet-eth deploy exec fixturenet-eth-bootnode-lighthouse /scripts/status-internal.sh +# echo "$(date +"%Y-%m-%d %T"): Getting stack status" +# $TEST_TARGET_SO --stack fixturenet-eth deploy exec fixturenet-eth-bootnode-lighthouse /scripts/status-internal.sh +echo "$(date +"%Y-%m-%d %T"): Getting initial block number" initial_block_number=$($TEST_TARGET_SO --stack fixturenet-eth deploy exec foundry "cast block-number") # Check that the block number increases some time later -sleep 12 +sleep 120 +echo "$(date +"%Y-%m-%d %T"): Getting subsequent block number" subsequent_block_number=$($TEST_TARGET_SO --stack fixturenet-eth deploy exec foundry "cast block-number") block_number_difference=$((subsequent_block_number - initial_block_number)) # Block height difference should be between 1 and some small number -if [[ $block_number_difference -gt 1 && $block_number_difference -lt 10 ]]; then +if [[ $block_number_difference -gt 1 && $block_number_difference -lt 100 ]]; then echo "Test passed" test_result=0 else echo "Test failed: block numbers were ${initial_block_number} and ${subsequent_block_number}" + echo "Logs from stack:" + $TEST_TARGET_SO --stack fixturenet-eth deploy logs test_result=1 fi $TEST_TARGET_SO --stack fixturenet-eth deploy down -echo "Removing cloned repositories" +echo "$(date +"%Y-%m-%d %T"): Removing cloned repositories" rm -rf $CERC_REPO_BASE_DIR +echo "$(date +"%Y-%m-%d %T"): Test finished" exit $test_result From f411590452c6f7d91b8ea7d278ba8186843a1bc1 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 17 Aug 2023 13:49:56 -0600 Subject: [PATCH 5/5] Tolerate missing deployment plugin source code files (#508) --- app/deployment_create.py | 76 +++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/app/deployment_create.py b/app/deployment_create.py index 16202284..2fbf2409 100644 --- a/app/deployment_create.py +++ b/app/deployment_create.py @@ -14,7 +14,6 @@ # along with this program. If not, see . import click -from dataclasses import dataclass from importlib import util import os from pathlib import Path @@ -27,6 +26,7 @@ from app.deploy_types import DeploymentContext, DeployCommandContext def _make_default_deployment_dir(): return "deployment-001" + def _get_ports(stack): ports = {} parsed_stack = get_parsed_stack_config(stack) @@ -42,6 +42,7 @@ def _get_ports(stack): ports[svc_name] = [ str(x) for x in svc["ports"] ] return ports + def _get_named_volumes(stack): # Parse the compose files looking for named volumes named_volumes = [] @@ -76,30 +77,30 @@ def _create_bind_dir_if_relative(volume, path_string, compose_dir): # See: https://stackoverflow.com/questions/45699189/editing-docker-compose-yml-with-pyyaml def _fixup_pod_file(pod, spec, compose_dir): - # Fix up volumes - if "volumes" in spec: - spec_volumes = spec["volumes"] - if "volumes" in pod: - pod_volumes = pod["volumes"] - for volume in pod_volumes.keys(): - if volume in spec_volumes: - volume_spec = spec_volumes[volume] - volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}" - _create_bind_dir_if_relative(volume, volume_spec, compose_dir) - new_volume_spec = {"driver": "local", - "driver_opts": { + # Fix up volumes + if "volumes" in spec: + spec_volumes = spec["volumes"] + if "volumes" in pod: + pod_volumes = pod["volumes"] + for volume in pod_volumes.keys(): + if volume in spec_volumes: + volume_spec = spec_volumes[volume] + volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}" + _create_bind_dir_if_relative(volume, volume_spec, compose_dir) + new_volume_spec = {"driver": "local", + "driver_opts": { "type": "none", "device": volume_spec_fixedup, "o": "bind" } - } - pod["volumes"][volume] = new_volume_spec - # Fix up ports - if "ports" in spec: - spec_ports = spec["ports"] - for container_name, container_ports in spec_ports.items(): - if container_name in pod["services"]: - pod["services"][container_name]["ports"] = container_ports + } + pod["volumes"][volume] = new_volume_spec + # Fix up ports + if "ports" in spec: + spec_ports = spec["ports"] + for container_name, container_ports in spec_ports.items(): + if container_name in pod["services"]: + pod["services"][container_name]["ports"] = container_ports def call_stack_deploy_init(deploy_command_context): @@ -107,10 +108,13 @@ def call_stack_deploy_init(deploy_command_context): # Call a function in it # If no function found, return None python_file_path = get_stack_file_path(deploy_command_context.stack).parent.joinpath("deploy", "commands.py") - spec = util.spec_from_file_location("commands", python_file_path) - imported_stack = util.module_from_spec(spec) - spec.loader.exec_module(imported_stack) - return imported_stack.init(deploy_command_context) + if python_file_path.exists(): + spec = util.spec_from_file_location("commands", python_file_path) + imported_stack = util.module_from_spec(spec) + spec.loader.exec_module(imported_stack) + return imported_stack.init(deploy_command_context) + else: + return None # TODO: fold this with function above @@ -119,10 +123,13 @@ def call_stack_deploy_setup(deploy_command_context, extra_args): # Call a function in it # If no function found, return None python_file_path = get_stack_file_path(deploy_command_context.stack).parent.joinpath("deploy", "commands.py") - spec = util.spec_from_file_location("commands", python_file_path) - imported_stack = util.module_from_spec(spec) - spec.loader.exec_module(imported_stack) - return imported_stack.setup(deploy_command_context, extra_args) + if python_file_path.exists(): + spec = util.spec_from_file_location("commands", python_file_path) + imported_stack = util.module_from_spec(spec) + spec.loader.exec_module(imported_stack) + return imported_stack.setup(deploy_command_context, extra_args) + else: + return None # TODO: fold this with function above @@ -131,10 +138,13 @@ def call_stack_deploy_create(deployment_context): # Call a function in it # If no function found, return None python_file_path = get_stack_file_path(deployment_context.command_context.stack).parent.joinpath("deploy", "commands.py") - spec = util.spec_from_file_location("commands", python_file_path) - imported_stack = util.module_from_spec(spec) - spec.loader.exec_module(imported_stack) - return imported_stack.create(deployment_context) + if python_file_path.exists(): + spec = util.spec_from_file_location("commands", python_file_path) + imported_stack = util.module_from_spec(spec) + spec.loader.exec_module(imported_stack) + return imported_stack.create(deployment_context) + else: + return None # Inspect the pod yaml to find config files referenced in subdirectories