kind-in-docker fixes (#77)

Reviewed-on: #77
Co-authored-by: David Boreham <david@bozemanpass.com>
Co-committed-by: David Boreham <david@bozemanpass.com>

act-runner/Dockerfile.task-executor

@@ -4,7 +4,7 @@ FROM ubuntu:22.04
 RUN ln -snf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && echo $CONTAINER_TIMEZONE > /etc/timezone
 
 # Install basic tools
-RUN apt update && apt install -y gpg curl apt-transport-https ca-certificates lsb-release build-essential
+RUN apt update && apt install -y gpg curl wget apt-transport-https ca-certificates lsb-release build-essential
 
 # Add Docker repo
 RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
@@ -16,7 +16,6 @@ ARG NODE_MAJOR=18
 # See: https://stackoverflow.com/a/77021599/1701505
 RUN set -uex; \
     apt-get update; \
-    apt-get install -y ca-certificates curl gnupg; \
     mkdir -p /etc/apt/keyrings; \
     curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
      | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg; \
@@ -29,5 +28,29 @@ RUN set -uex; \
 RUN apt update && apt install -y docker-ce && rm -rf /var/lib/apt/lists/*
 # Install sudo because some actions projects assume it is present, and it is present in GitHub runners
 RUN apt update && apt install -y sudo
+# Make sure we have some other basic tools that scripts expect.
+RUN apt update && apt install -y wget curl jq
 # Install software-properties-common so we have the add-apt-repository command, used by some actions to add a package repo
 RUN apt update && apt install -y software-properties-common
+
+# Packages and files to support dind functionality see: https://github.com/cruizba/ubuntu-dind
+RUN apt update && apt install -y iptables supervisor
+
+COPY modprobe start-docker.sh entrypoint.sh /usr/local/bin/
+COPY supervisor/ /etc/supervisor/conf.d/
+COPY logger.sh /opt/bash-utils/logger.sh
+COPY cgroup-helper.sh /opt/bash-utils/cgroup-helper.sh
+
+RUN chmod +x /usr/local/bin/start-docker.sh \
+	/usr/local/bin/entrypoint.sh \
+	/usr/local/bin/modprobe
+
+ENV DOCKER_HOST "unix:///var/run/dind.sock"
+
+# This VOLUME directive is required for k3d to work, probably because it needs the directory to exist
+# the volume does not need to be mounted.
+VOLUME /var/lib/docker
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
+CMD ["bash"]
+

act-runner/cgroup-helper.sh

@@ -0,0 +1,34 @@
+
+# This file needs to be source'ed and the function join_cgroup called, by any script that goes on to run kind
+# This is required due to issues with properly virtualizing the cgroup hierarchy that exist at present in docker
+# See: https://github.com/earthly/earthly/blob/main/buildkitd/dockerd-wrapper.sh#L56
+function configure_cgroup() {
+    if [ -f "/sys/fs/cgroup/cgroup.controllers" ]; then
+        echo >&2 "INFO: detected cgroup v2, configuring nested docker group"
+
+        local cgroup_name="nested-dockerd" # NOTE: has to be the same as the function below (local var to prevent overriding in the caller)
+
+        # move script to separate cgroup, to prevent the root cgroup from becoming threaded (which will prevent systemd images (e.g. kind) from running)
+        mkdir /sys/fs/cgroup/${cgroup_name}
+        echo $$ > /sys/fs/cgroup/${cgroup_name}/cgroup.procs
+
+       # This script is run from inside entrypoint.sh
+       # so we also need to move the parent pid into this new group, which is weird
+       # TODO: we should unwrap this so $$ is all we need to move
+        echo 1 > /sys/fs/cgroup/${cgroup_name}/cgroup.procs
+
+        if [ "$(wc -l < /sys/fs/cgroup/cgroup.procs)" != "0" ]; then
+            echo >&2 "WARNING: processes exist in the root cgroup; this may cause errors during cgroup initialization"
+        fi
+
+        root_cgroup_type="$(cat /sys/fs/cgroup/cgroup.type)"
+        if [ "$root_cgroup_type" != "domain" ]; then
+            echo >&2 "WARNING: expected cgroup type of \"domain\", but got \"$root_cgroup_type\" instead"
+        fi
+    fi
+}
+
+function join_cgroup() {
+    local cgroup_name="nested-dockerd" # NOTE: has to be the same as the function above (local var to prevent overriding in the caller)
+    echo $$ > /sys/fs/cgroup/${cgroup_name}/cgroup.procs
+}

act-runner/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Start docker
+start-docker.sh
+
+# Execute specified command
+"$@"

act-runner/logger.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# Logger from this post http://www.cubicrace.com/2016/03/log-tracing-mechnism-for-shell-scripts.html
+
+function INFO(){
+    local function_name="${FUNCNAME[1]}"
+    local msg="$1"
+    timeAndDate=`date`
+    echo "[$timeAndDate] [INFO] [${0}] $msg"
+}
+
+
+function DEBUG(){
+    local function_name="${FUNCNAME[1]}"
+    local msg="$1"
+    timeAndDate=`date`
+    echo "[$timeAndDate] [DEBUG] [${0}] $msg"
+}
+
+function ERROR(){
+    local function_name="${FUNCNAME[1]}"
+    local msg="$1"
+    timeAndDate=`date`
+    echo "[$timeAndDate] [ERROR]  $msg"
+}

act-runner/modprobe

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -eu
+
+# "modprobe" without modprobe
+# https://twitter.com/lucabruno/status/902934379835662336
+
+# this isn't 100% fool-proof, but it'll have a much higher success rate than simply using the "real" modprobe
+
+# Docker often uses "modprobe -va foo bar baz"
+# so we ignore modules that start with "-"
+for module; do
+	if [ "${module#-}" = "$module" ]; then
+		ip link show "$module" || true
+		lsmod | grep "$module" || true
+	fi
+done
+
+# remove /usr/local/... from PATH so we can exec the real modprobe as a last resort
+export PATH='/usr/sbin:/usr/bin:/sbin:/bin'
+exec modprobe "$@"

act-runner/stack/deploy/commands.py

@@ -21,6 +21,7 @@ def create(context, extra_args):
     # Our goal here is just to copy the config file for act
     deployment_config_dir = context.deployment_dir.joinpath("data",
                                                             "act-runner-config")
-    compose_file = [f for f in context.command_context.cluster_context.compose_files if "act-runner" in f][0]
+    command_context = extra_args[2]
+    compose_file = [f for f in command_context.cluster_context.compose_files if "act-runner" in f][0]
     source_config_file = Path(compose_file).parent.joinpath("config", "act-runner-config.yml")
     copy(source_config_file, deployment_config_dir)

act-runner/start-docker.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+source /opt/bash-utils/logger.sh
+source /opt/bash-utils/cgroup-helper.sh
+
+function wait_for_process () {
+    local max_time_wait=30
+    local process_name="$1"
+    local waited_sec=0
+    while ! pgrep "$process_name" >/dev/null && ((waited_sec < max_time_wait)); do
+        INFO "Process $process_name is not running yet. Retrying in 1 seconds"
+        INFO "Waited $waited_sec seconds of $max_time_wait seconds"
+        sleep 1
+        ((waited_sec=waited_sec+1))
+        if ((waited_sec >= max_time_wait)); then
+            return 1
+        fi
+    done
+    return 0
+}
+
+# Some payloads (e.g. kind) need systemd to run, which in turn requires forking the cgroup hierarchy
+configure_cgroup
+
+INFO "Starting supervisor"
+/usr/bin/supervisord -n >> /dev/null 2>&1 &
+
+INFO "Waiting for docker to be running"
+wait_for_process dockerd
+if [ $? -ne 0 ]; then
+    ERROR "dockerd is not running after max time"
+    exit 1
+else
+    INFO "dockerd is running"
+fi

act-runner/supervisor/dockerd.conf

@@ -0,0 +1,6 @@
+[program:dockerd]
+command=/usr/bin/dockerd -H %(ENV_DOCKER_HOST)s --userland-proxy=false
+autostart=true
+autorestart=true
+stderr_logfile=/var/log/dockerd.err.log
+stdout_logfile=/var/log/dockerd.out.log

gitea/docker-compose.yml

@@ -1,7 +1,7 @@
 
 services:
   server:
-    image: gitea/gitea:1.21.0-rc1
+    image: gitea/gitea:1.21
     environment:
       - USER_UID=1000
       - USER_GID=1000