Use containerd-snapshotter image store in task executor

Reviewed-on: #77
Co-authored-by: David Boreham <david@bozemanpass.com>
Co-committed-by: David Boreham <david@bozemanpass.com>

act-runner/Dockerfile.task-executor

@@ -0,0 +1,57 @@
+FROM ubuntu:22.04
+
+# Set system time zone to prevent the tzdata package from hanging looking for user input
+RUN ln -snf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && echo $CONTAINER_TIMEZONE > /etc/timezone
+
+# Install basic tools
+RUN apt update && apt install -y gpg curl wget apt-transport-https ca-certificates lsb-release build-essential
+
+# Add Docker repo
+RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+ARG NODE_MAJOR=18
+
+# Add NodeJS repo
+# See: https://stackoverflow.com/a/77021599/1701505
+RUN set -uex; \
+    apt-get update; \
+    mkdir -p /etc/apt/keyrings; \
+    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+     | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg; \
+    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" \
+     > /etc/apt/sources.list.d/nodesource.list; \
+    apt-get update; \
+    apt-get install nodejs -y;
+
+# Install Docker
+RUN apt update && apt install -y docker-ce && rm -rf /var/lib/apt/lists/*
+# Install sudo because some actions projects assume it is present, and it is present in GitHub runners
+RUN apt update && apt install -y sudo
+# Make sure we have some other basic tools that scripts expect.
+RUN apt update && apt install -y wget curl jq
+# Install software-properties-common so we have the add-apt-repository command, used by some actions to add a package repo
+RUN apt update && apt install -y software-properties-common
+
+# Packages and files to support dind functionality see: https://github.com/cruizba/ubuntu-dind
+RUN apt update && apt install -y iptables supervisor
+
+COPY modprobe start-docker.sh entrypoint.sh /usr/local/bin/
+COPY supervisor/ /etc/supervisor/conf.d/
+COPY logger.sh /opt/bash-utils/logger.sh
+COPY cgroup-helper.sh /opt/bash-utils/cgroup-helper.sh
+COPY task-executor-docker-daemon.json /etc/docker/daemon.json
+
+RUN chmod +x /usr/local/bin/start-docker.sh \
+	/usr/local/bin/entrypoint.sh \
+	/usr/local/bin/modprobe
+
+ENV DOCKER_HOST "unix:///var/run/dind.sock"
+
+# This VOLUME directive is required for k3d to work, probably because it needs the directory to exist
+# the volume does not need to be mounted.
+VOLUME /var/lib/docker
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
+CMD ["bash"]
+

act-runner/cgroup-helper.sh

@@ -0,0 +1,34 @@
+
+# This file needs to be source'ed and the function join_cgroup called, by any script that goes on to run kind
+# This is required due to issues with properly virtualizing the cgroup hierarchy that exist at present in docker
+# See: https://github.com/earthly/earthly/blob/main/buildkitd/dockerd-wrapper.sh#L56
+function configure_cgroup() {
+    if [ -f "/sys/fs/cgroup/cgroup.controllers" ]; then
+        echo >&2 "INFO: detected cgroup v2, configuring nested docker group"
+
+        local cgroup_name="nested-dockerd" # NOTE: has to be the same as the function below (local var to prevent overriding in the caller)
+
+        # move script to separate cgroup, to prevent the root cgroup from becoming threaded (which will prevent systemd images (e.g. kind) from running)
+        mkdir /sys/fs/cgroup/${cgroup_name}
+        echo $$ > /sys/fs/cgroup/${cgroup_name}/cgroup.procs
+
+       # This script is run from inside entrypoint.sh
+       # so we also need to move the parent pid into this new group, which is weird
+       # TODO: we should unwrap this so $$ is all we need to move
+        echo 1 > /sys/fs/cgroup/${cgroup_name}/cgroup.procs
+
+        if [ "$(wc -l < /sys/fs/cgroup/cgroup.procs)" != "0" ]; then
+            echo >&2 "WARNING: processes exist in the root cgroup; this may cause errors during cgroup initialization"
+        fi
+
+        root_cgroup_type="$(cat /sys/fs/cgroup/cgroup.type)"
+        if [ "$root_cgroup_type" != "domain" ]; then
+            echo >&2 "WARNING: expected cgroup type of \"domain\", but got \"$root_cgroup_type\" instead"
+        fi
+    fi
+}
+
+function join_cgroup() {
+    local cgroup_name="nested-dockerd" # NOTE: has to be the same as the function above (local var to prevent overriding in the caller)
+    echo $$ > /sys/fs/cgroup/${cgroup_name}/cgroup.procs
+}

act-runner/config/act-runner-config.yml

@@ -0,0 +1,50 @@
+# Example configuration file, it's safe to copy this as the default config file without any modification.
+
+log:
+  # The level of logging, can be trace, debug, info, warn, error, fatal
+  level: info
+
+runner:
+  # Where to store the registration result.
+  file: /data/.runner
+  # Execute how many tasks concurrently at the same time.
+  capacity: 1
+  # # Extra environment variables to run jobs.
+  # envs:
+  #   A_TEST_ENV_NAME_1: a_test_env_value_1
+  #   A_TEST_ENV_NAME_2: a_test_env_value_2
+  # # Extra environment variables to run jobs from a file.
+  # # It will be ignored if it's empty or the file doesn't exist.
+  # env_file: .env
+  # # The timeout for a job to be finished.
+  # # Please note that the Gitea instance also has a timeout (3h by default) for the job.
+  # # So the job could be stopped by the Gitea instance if it's timeout is shorter than this.
+  timeout: 3h
+  # Whether skip verifying the TLS certificate of the Gitea instance.
+  insecure: false
+  # The timeout for fetching the job from the Gitea instance.
+  fetch_timeout: 5s
+  # The interval for fetching the job from the Gitea instance.
+  fetch_interval: 2s
+
+cache:
+  # Enable cache server to use actions/cache.
+  enabled: true
+  # The directory to store the cache data.
+  # If it's empty, the cache data will be stored in $HOME/.cache/actcache.
+  dir: ""
+  # The host of the cache server.
+  # It's not for the address to listen, but the address to connect from job containers.
+  # So 0.0.0.0 is a bad choice, leave it empty to detect automatically.
+  host: "gitea.local"
+  # The port of the cache server.
+  # 0 means to use a random available port.
+  port: 8088
+
+container:
+  # Whether to use privileged mode or not when launching task containers (privileged mode is required for Docker-in-Docker).
+  privileged: true
+  # And other options to be used when the container is started (eg, --add-host=my.gitea.url:host-gateway).
+  options: --add-host=gitea.local:host-gateway --volume "/var/lib/docker"
+  valid_volumes:
+    - act-runner-shared

act-runner/docker-compose.yml

@@ -0,0 +1,23 @@
+
+services:
+  runner:
+    image: cerc/act-runner:local
+    restart: always
+    environment:
+      - CONFIG_FILE=/config/act-runner-config.yml
+      # Note: eMdEwIzSo87nBh0UFWZlbp308j6TNWr3WhWxQqIc is a static token we use for convenience in stand-alone deployments. Not secure, obviously.
+      - GITEA_RUNNER_REGISTRATION_TOKEN=${CERC_GITEA_RUNNER_REGISTRATION_TOKEN:-eMdEwIzSo87nBh0UFWZlbp308j6TNWr3WhWxQqIc}
+      - GITEA_INSTANCE_URL=${CERC_GITEA_INSTANCE_URL:-http://gitea.local:3000}
+      - GITEA_RUNNER_LABELS=${CERC_GITEA_RUNNER_LABELS:-ubuntu-latest:docker://cerc/act-runner-task-executor:local,ubuntu-22.04:docker://cerc/act-runner-task-executor:local}
+    extra_hosts:
+      - "gitea.local:host-gateway"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - act-runner-data:/data
+      - act-runner-config:/config:ro
+    ports:
+      - 8088
+
+volumes:
+  act-runner-data:
+  act-runner-config:

act-runner/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Start docker
+start-docker.sh
+
+# Execute specified command
+"$@"

act-runner/logger.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# Logger from this post http://www.cubicrace.com/2016/03/log-tracing-mechnism-for-shell-scripts.html
+
+function INFO(){
+    local function_name="${FUNCNAME[1]}"
+    local msg="$1"
+    timeAndDate=`date`
+    echo "[$timeAndDate] [INFO] [${0}] $msg"
+}
+
+
+function DEBUG(){
+    local function_name="${FUNCNAME[1]}"
+    local msg="$1"
+    timeAndDate=`date`
+    echo "[$timeAndDate] [DEBUG] [${0}] $msg"
+}
+
+function ERROR(){
+    local function_name="${FUNCNAME[1]}"
+    local msg="$1"
+    timeAndDate=`date`
+    echo "[$timeAndDate] [ERROR]  $msg"
+}

act-runner/modprobe

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -eu
+
+# "modprobe" without modprobe
+# https://twitter.com/lucabruno/status/902934379835662336
+
+# this isn't 100% fool-proof, but it'll have a much higher success rate than simply using the "real" modprobe
+
+# Docker often uses "modprobe -va foo bar baz"
+# so we ignore modules that start with "-"
+for module; do
+	if [ "${module#-}" = "$module" ]; then
+		ip link show "$module" || true
+		lsmod | grep "$module" || true
+	fi
+done
+
+# remove /usr/local/... from PATH so we can exec the real modprobe as a last resort
+export PATH='/usr/sbin:/usr/bin:/sbin:/bin'
+exec modprobe "$@"

act-runner/post_start.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi

act-runner/pre_start.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi

act-runner/stack/deploy/commands.py

@@ -0,0 +1,27 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from pathlib import Path
+from shutil import copy
+
+
+def create(context, extra_args):
+    # Our goal here is just to copy the config file for act
+    deployment_config_dir = context.deployment_dir.joinpath("data",
+                                                            "act-runner-config")
+    command_context = extra_args[2]
+    compose_file = [f for f in command_context.cluster_context.compose_files if "act-runner" in f][0]
+    source_config_file = Path(compose_file).parent.joinpath("config", "act-runner-config.yml")
+    copy(source_config_file, deployment_config_dir)

act-runner/start-docker.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+source /opt/bash-utils/logger.sh
+source /opt/bash-utils/cgroup-helper.sh
+
+function wait_for_process () {
+    local max_time_wait=30
+    local process_name="$1"
+    local waited_sec=0
+    while ! pgrep "$process_name" >/dev/null && ((waited_sec < max_time_wait)); do
+        INFO "Process $process_name is not running yet. Retrying in 1 seconds"
+        INFO "Waited $waited_sec seconds of $max_time_wait seconds"
+        sleep 1
+        ((waited_sec=waited_sec+1))
+        if ((waited_sec >= max_time_wait)); then
+            return 1
+        fi
+    done
+    return 0
+}
+
+# Some payloads (e.g. kind) need systemd to run, which in turn requires forking the cgroup hierarchy
+configure_cgroup
+
+INFO "Starting supervisor"
+/usr/bin/supervisord -n >> /dev/null 2>&1 &
+
+INFO "Waiting for docker to be running"
+wait_for_process dockerd
+if [ $? -ne 0 ]; then
+    ERROR "dockerd is not running after max time"
+    exit 1
+else
+    INFO "dockerd is running"
+fi

act-runner/supervisor/dockerd.conf

@@ -0,0 +1,6 @@
+[program:dockerd]
+command=/usr/bin/dockerd -H %(ENV_DOCKER_HOST)s --userland-proxy=false
+autostart=true
+autorestart=true
+stderr_logfile=/var/log/dockerd.err.log
+stdout_logfile=/var/log/dockerd.out.log

act-runner/task-executor-docker-daemon.json

@@ -0,0 +1,5 @@
+{
+  "features": {
+    "containerd-snapshotter": true
+  }
+}

gitea/README.md

@@ -1,3 +1,32 @@
-## Manual deployment notes
+## Deployment Notes
 ### Gitea
-1. Create admin user in web UI on initial setup
+
+#### Build gitea/act_runner Docker Container
+1. To build the `act_runner` container from Gitea, in another directory run:
+```
+git clone https://gitea.com/gitea/act_runner
+cd act_runner
+docker build -t cerc/act-runner:local .
+```
+
+#### Deploy Gitea Stack
+1. `cd ./gitea`
+1. Build the task executor container: `docker build -t cerc/act-runner-task-executor:local -f Dockerfile.task-executor .`
+1. Run the script `./run-this-first.sh`
+1. Bring up the gitea cluster `docker compose up -d`
+1. Run the script `./initialize-gitea.sh`
+1. Note the access token printed, it will be needed to publish packages.
+
+#### Debugging
+Gitea server logs can be seen via docker logs <container-id>.
+To enable more verbose log output add an environment variable definition like:
+```
+GITEA__log__LEVEL=TRACE
+```
+to the `server` definition in `docker-compose.yml` and re-start.
+Details on how to setup remote debugging of the gitea server inside its container can be found [here](gitea-debugging.md).
+
+#### Action Runners
+
+A Dockerized action runner is deployed by default for the labels `ubuntu-latest` and `ubuntu-22.04`.  Details on deploying
+additional runners can be found [here](../act-runner/act-runner.md).

gitea/act-runner.md

@@ -0,0 +1,68 @@
+## Deploying Action Runners
+
+IMPORTANT NOTE: you should be aware that anyone with the ability to modify code run under a CI job in the host Gitea (this includes anyone with commit rights; anyone with the ability to modify any dependency; anyone with the ability to modify dependent components such as base container images) CAN POTENTIALLY COMPROMISE (hack, take over, steal data from) the machine hosting a runner. Proceed with caution.
+
+### Releases
+Gitea publishes binary releases of [gitea/act_runner](https://gitea.com/gitea/act_runner/releases) for many platform and architectures, which can be used to deploy new action runners simply.
+
+The following example uses `gitea/act_runner` 0.2.6 to deploy a runner on macOS Ventura 13.3 x64.
+
+### Registration Token
+
+> Note: Runners can be registered globally for an entire Gitea instance, for a specific organization, or for a single repo.  This example registers globally.
+
+Before executing the runner, first obtain a registration token by visiting http://gitea.local:3000/admin/actions/runners, clicking the 'Create new Runner' button, and copying the displayed
+registration token, for example, `FTyMBkcK9ErmD0wm8LfBzfXOUUlQA7dBJF6BB64Z`.
+
+### Runner Registration and Startup
+
+After you have obtained a registration token, download the `gitea/act_runner` release matching your platform and architecture and run it as follows:
+
+```
+# Download latest gitea/act_runner release for your platform.
+$ wget https://gitea.com/gitea/act_runner/releases/download/latest/act_runner-0.2.6-darwin-amd64 && chmod a+x act_runner-0.2.6-darwin-amd64
+
+# Register the runner with the Gitea instance using the token obtained above.
+$ ./act_runner-0.2.6-darwin-amd64 register \
+    --instance http://gitea.local:3000 \
+    --labels 'darwin-latest-amd64:host,darwin-13-amd64:host' \
+    --name 'darwin-amd64-001' \
+    --token "FTyMBkcK9ErmD0wm8LfBzfXOUUlQA7dBJF6BB64Z" \
+    --no-interactive
+
+# Launch it in daemon mode, waiting for jobs.
+$ ./act_runner-0.2.6-darwin-amd64 daemon
+```
+
+### Labels
+
+The most important detail in this example is the label.  For the Ubuntu runner which is deployed automatically with this project, the label `ubuntu-latest:docker://cerc/act-runner-task-executor:local` is
+used, which instructs `gitea/act_runner` that a task which `runs-on: ubuntu-latest` should be executed inside an instance of the `cerc/act-runner-task-executor:local` Docker container.  In this example, the label is `darwin-latest-amd64:host`.  This means that a task which `runs-on: darwin-latest-amd64` will be executed natively on the host machine.  Since there are additional security implications when executing tasks
+on the host, only trusted repositories with strict access controls should be allowed to schedule CI jobs on the runner.
+
+### Example Workflow
+
+This very simple workflow will schedule jobs on both macOS (`darwin-latest-amd64`) and Linux (`ubuntu-latest`) runners.
+
+```
+name: macOS test
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  test-macos:
+    name: "Run on macOS"
+    runs-on: darwin-latest-amd64
+    steps:
+      - name: "uname"
+        run: uname -a
+  test-linux:
+    name: "Run on Ubuntu"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "uname"
+        run: uname -a
+```

gitea/delete-repo.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Script that calls the Giteap API to delete one repo
+
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+
+if ! [[ $# -eq 1 ]]; then
+    echo "Illegal number of parameters" >&2
+    exit 1
+fi
+repo_to_delete=$1
+
+if [[ -z "${CERC_GITEA_AUTH_TOKEN}" ]]; then
+    echo "CERC_GITEA_AUTH_TOKEN is not set" >&2
+    exit 1
+fi
+if [[ -z "${CERC_GITEA_API_URL}" ]]; then
+    echo "CERC_GITEA_API_URL is not set" >&2
+    exit 1
+fi
+if [[ "${CERC_GITEA_MIRROR_REPO}" == "true" ]]; then
+    is_mirror=true
+else
+    is_mirror=false
+fi
+
+gitea_target_org=$(dirname ${repo_to_delete})
+gitea_target_repo_name=$(basename ${repo_to_delete})
+# Sanity check the repo name
+if [[ -z "${gitea_target_org}" ]]; then
+    echo "${repo_to_delete} is not a valid repo name" >&2
+    exit 1
+fi
+if [[ -z "${gitea_target_repo_name}" ]]; then
+    echo "${repo_to_delete} is not a valid repo name" >&2
+    exit 1
+fi
+
+echo "****** DELETING repo: ${repo_to_delete}"
+# Note use: --trace-ascii - \ below to see the raw request
+delete_response=$( curl -s -X DELETE "${CERC_GITEA_API_URL}/api/v1/repos/${repo_to_delete}" \
+      -H "Authorization: token ${CERC_GITEA_AUTH_TOKEN}" \
+      -H  "accept: application/json" \
+)
+echo ${delete_response} | jq -r

gitea/docker-compose.yml

@@ -1,13 +1,7 @@
-version: "3"
-
-networks:
-  gitea:
-    external: false
 
 services:
   server:
-    image: gitea/gitea:1.17.1
-    container_name: gitea
+    image: gitea/gitea:1.21
     environment:
       - USER_UID=1000
       - USER_GID=1000
@@ -19,11 +13,13 @@ services:
       - GITEA__server__HTTP_PORT=3000
       - GITEA__server__LOCAL_ROOT_URL=http://gitea.local:3000/
       - GITEA__server__ROOT_URL=http://gitea.local:3000/
+      - GITEA__actions__ENABLED=true
+      - GITEA__security__INSTALL_LOCK=true
     restart: always
-    networks:
-      - gitea
+    extra_hosts:
+      - "gitea.local:host-gateway"
     volumes:
-      - ./gitea:/data
+      - gitea-data:/data
       - /etc/timezone:/etc/timezone:ro
       - /etc/localtime:/etc/localtime:ro
     # TODO: remove fixed host port number
@@ -40,7 +36,12 @@ services:
       - POSTGRES_USER=gitea
       - POSTGRES_PASSWORD=gitea
       - POSTGRES_DB=gitea
-    networks:
-      - gitea
+    # Workaround below for lack of docker uid mapping. Change the container's postgres user's uid/gid to match the host user's
+    entrypoint: bash
+    command: -c  'usermod -u ${CERC_HOST_UID:-1000} postgres;groupmod -g ${CERC_HOST_GID:-1000} postgres;exec /usr/local/bin/docker-entrypoint.sh postgres'
     volumes:
-      - ./postgres:/var/lib/postgresql/data
+      - postgres-data:/var/lib/postgresql/data
+
+volumes:
+  gitea-data:
+  postgres-data:

gitea/gitea-debugging.md

@@ -0,0 +1,175 @@
+## Notes using a debugger with Gitea
+
+### Changes to Gitea
+Assuming the Gitea repository cloned at `/path/to/gitea` (adjust below for your actual location),
+make the following changes to a cloned gitea repository then build a new container with:
+```
+$ docker build -t my-org/gitea:debug -f Dockerfile .
+```
+
+Gitea project changes:
+Dockerfile adds delve debugger binary, adjust compiler flags to suit debugging, expose port 40000 for remote debugging.
+```
+diff --git a/Dockerfile b/Dockerfile
+index 06481cdf5..a49fd0266 100644
+--- a/Dockerfile
++++ b/Dockerfile
+@@ -16,6 +16,10 @@ RUN apk --no-cache add build-base git nodejs npm
+ COPY . ${GOPATH}/src/code.gitea.io/gitea
+ WORKDIR ${GOPATH}/src/code.gitea.io/gitea
+
++RUN go install github.com/go-delve/delve/cmd/dlv@latest
++
++ENV EXTRA_GOFLAGS '-gcflags="all=-N -l"'
++
+ #Checkout version if set
+ RUN if [ -n "${GITEA_VERSION}" ]; then git checkout "${GITEA_VERSION}"; fi \
+  && make clean-all build
+@@ -26,7 +30,7 @@ RUN go build contrib/environment-to-ini/environment-to-ini.go
+ FROM docker.io/library/alpine:3.18
+ LABEL maintainer="maintainers@gitea.io"
+
+-EXPOSE 22 3000
++EXPOSE 22 3000 40000
+
+ RUN apk --no-cache add \
+     bash \
+@@ -65,6 +69,7 @@ COPY docker/root /
+ COPY --from=build-env /go/src/code.gitea.io/gitea/gitea /app/gitea/gitea
+ COPY --from=build-env /go/src/code.gitea.io/gitea/environment-to-ini /usr/local/bin/environment-to-ini
+ COPY --from=build-env /go/src/code.gitea.io/gitea/contrib/autocompletion/bash_autocomplete /etc/profile.d/gitea_bash_autocomplete.sh
++COPY --from=build-env  /go/bin/dlv /usr/local/bin/
+ RUN chmod 755 /usr/bin/entrypoint /app/gitea/gitea /usr/local/bin/gitea /usr/local/bin/environment-to-ini
+ RUN chmod 755 /etc/s6/gitea/* /etc/s6/openssh/* /etc/s6/.s6-svscan/*
+ RUN chmod 644 /etc/profile.d/gitea_bash_autocomplete.sh
+ ```
+Makefile removes linker flags that strip symbols:
+ ```
+diff --git a/Makefile b/Makefile
+index 16841796b..35fdaf1de 100644
+--- a/Makefile
++++ b/Makefile
+@@ -789,7 +789,7 @@ check: test
+
+ .PHONY: install $(TAGS_PREREQ)
+ install: $(wildcard *.go)
+-       CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) install -v -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)'
++       CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) install -v -tags '$(TAGS)' -ldflags '$(LDFLAGS)'
+
+ .PHONY: build
+ build: frontend backend
+@@ -817,7 +817,7 @@ security-check:
+        go run $(GOVULNCHECK_PACKAGE) ./...
+
+ $(EXECUTABLE): $(GO_SOURCES) $(TAGS_PREREQ)
+-       CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)' -o $@
++       CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $@
+
+ .PHONY: release
+ release: frontend generate release-windows release-linux release-darwin release-freebsd release-copy release-compress vendor release-sources release-docs release-check
+```
+run script inserts delve as the executed binary, with appropriate commands for it to spawn the gitea binary on startup:
+``` 
+diff --git a/docker/root/etc/s6/gitea/run b/docker/root/etc/s6/gitea/run
+index 7b858350f..26bd2eeb3 100755
+--- a/docker/root/etc/s6/gitea/run
++++ b/docker/root/etc/s6/gitea/run
+@@ -1,6 +1,25 @@
+ #!/bin/bash
++if [ -n "$CERC_SCRIPT_DEBUG" ]; then
++    set -x
++fi
++
+ [[ -f ./setup ]] && source ./setup
+
++GITEA="/app/gitea/gitea"
++WORK_DIR="/app/gitea"
++CUSTOM_PATH="/data/gitea"
++
++# Provide docker defaults
++export GITEA_WORK_DIR="${GITEA_WORK_DIR:-$WORK_DIR}"
++export GITEA_CUSTOM="${GITEA_CUSTOM:-$CUSTOM_PATH}"
++
++# exec -a "$0" "$GITEA" $CONF_ARG "$@"
++
++START_CMD="/usr/local/bin/gitea"
++if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then
++    START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec "$GITEA"  --continue --"
++fi
++
+ pushd /app/gitea >/dev/null
+-exec su-exec $USER /usr/local/bin/gitea web
++exec su-exec $USER $START_CMD $CONF_ARG web
+ popd
+```
+
+### Changes to the compose config
+
+1. Specify the newly build container image.
+1. Enable remote debugging with `CERC_REMOTE_DEBUG=true`
+1. Enable trace logging with `GITEA__log__LEVEL=Trace`
+1. Mount the project source into the container (path must be the same absolute path as on the host)
+1. Map the go debug port (40000 in this case) into the host.
+
+```
+diff --git a/gitea/docker-compose.yml b/gitea/docker-compose.yml
+index 59fea80..35feed0 100644
+--- a/gitea/docker-compose.yml
++++ b/gitea/docker-compose.yml
+@@ -1,8 +1,9 @@
+
+ services:
+   server:
+-    image: gitea/gitea:1.19.3
++    image: my-org/gitea:debug
+     environment:
++      - CERC_REMOTE_DEBUG=true
+       - USER_UID=1000
+       - USER_GID=1000
+       - GITEA__database__DB_TYPE=postgres
+@@ -15,6 +16,7 @@ services:
+       - GITEA__server__ROOT_URL=http://gitea.local:3000/
+       - GITEA__actions__ENABLED=true
+       - GITEA__security__INSTALL_LOCK=true
++      - GITEA__log__LEVEL=Trace
+     restart: always
+     extra_hosts:
+       - "gitea.local:host-gateway"
+@@ -22,10 +24,12 @@ services:
+       - ./gitea:/data
+       - /etc/timezone:/etc/timezone:ro
+       - /etc/localtime:/etc/localtime:ro
++      - /path/to/gitea:/path/to/gitea:ro
+     # TODO: remove fixed host port number
+     ports:
+       - "3000:3000"
+       - "222:22"
++      - "40000:40000"
+     depends_on:
+       - db
+```
+### Debug with VSCode
+Use a `launch.json` file like this:
+```
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Container gitea",
+            "type": "go",
+            "request": "attach",
+            "mode": "remote",
+            "remotePath": "/path/to/gitea",
+            "port": 40000,
+            "host": "127.0.0.1",
+            "substitutePath": [
+                { "from": "/path/to/gitea", "to": "/go/src/code.gitea.io/gitea" }
+            ]
+        }
+    ]
+}
+```
+With the gitea container running it should now be possible to "Run with debugging" and set breakpoints in the source code. If the breakpoints are not solid dots, something is wrong.

gitea/initialize-gitea.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# Run this script once after bringing up gitea in docker compose
+# TODO: add a check to detect that gitea has not fully initialized yet (no user relation error)
+
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+
+secure_password() {
+    # use openssl as the source, because it behaves similarly on both linux and macos
+    # we generate extra bytes so that even if tr deletes some chars we will still have plenty
+    openssl rand -base64 32 | tr -d '\/+=' | head -c 10 && echo
+}
+
+GITEA_USER=${CERC_GITEA_NEW_ADMIN_USERNAME:-"gitea_admin"}
+GITEA_PASSWORD=${CERC_GITEA_SET_NEW_ADMIN_PASSWORD:-"$(secure_password)"}
+GITEA_USER_EMAIL=${CERC_GITEA_SET_NEW_ADMIN_EMAIL:-${GITEA_USER}@example.com}
+GITEA_NEW_ORGANIZATION=${CERC_GITEA_NEW_ORGANIZATION:-"cerc-io"}
+GITEA_URL_PREFIX=http://localhost:3000
+CERC_GITEA_TOKEN_NAME=laconic-so-publication-token
+
+if ! [[ -n "$CERC_GITEA_RUNNER_REGISTRATION_TOKEN" ]]; then
+    echo "Warning: using insecure default runner registration token"
+    CERC_GITEA_RUNNER_REGISTRATION_TOKEN=eMdEwIzSo87nBh0UFWZlbp308j6TNWr3WhWxQqIc
+fi
+
+# Create admin user
+# First check if it already exists
+if [[ -z ${CERC_SO_COMPOSE_PROJECT} ]] ; then
+    compose_command="docker compose"
+else
+    compose_command="docker compose -p ${CERC_SO_COMPOSE_PROJECT}"
+fi
+sleep 15
+${compose_command} exec --user git server gitea admin user list --admin | grep -v -e "^ID" | awk '{ print $2 }' | grep ${GITEA_USER} > /dev/null
+if [[ $? == 1 ]] ; then
+    # Then create if it wasn't found
+    ${compose_command} exec --user git server gitea admin user create --admin --username ${GITEA_USER} --password ${GITEA_PASSWORD} --email ${GITEA_USER_EMAIL}
+fi
+# HACK: sleep a bit because if we don't gitea will return empty responses
+sleep 5
+# Check if the token already exists
+token_response=$( curl -s "${GITEA_URL_PREFIX}/api/v1/users/${GITEA_USER}/tokens" \
+  -u ${GITEA_USER}:${GITEA_PASSWORD} \
+  -H "Content-Type: application/json")
+if [[ -n ${token_response} ]] ; then
+    # Simple check for re-running this script. Ideally we should behave more elegantly.
+    if [[ "${token_response}" == *"password is invalid"* ]]; then
+        echo "Note: admin password is invalid, skipping subsqeuent steps"
+        exit 0
+    fi
+    echo ${token_response}  | jq --exit-status -r 'to_entries[] | select(.value.name == "'${CERC_GITEA_TOKEN_NAME}'")'
+    if [[ $? == 0 ]] ; then
+        token_found=1
+    fi
+fi
+if [[ ${token_found} != 1 ]] ; then
+    # Create access token if not found
+    # Note that we either create the token here, or we needed to be passed 
+    # the token by the caller. This is because gitea won't release the token
+    # plaintext post-creation.
+    new_gitea_token=$( curl -s -X POST "${GITEA_URL_PREFIX}/api/v1/users/${GITEA_USER}/tokens" \
+      -u ${GITEA_USER}:${GITEA_PASSWORD} \
+      -H "Content-Type: application/json" \
+      -d '{"name":"'${CERC_GITEA_TOKEN_NAME}'", "scopes": [ "read:admin", "write:admin", "read:organization", "write:organization", "read:repository", "write:repository", "read:package", "write:package" ] }' \
+      | jq -r .sha1 )
+    echo "NOTE: This is your gitea access token: ${new_gitea_token}. Keep it safe and secure, it can not be fetched again from gitea."
+    echo "NOTE: To use with laconic-so set this environment variable: export CERC_NPM_AUTH_TOKEN=${new_gitea_token}"
+    CERC_GITEA_AUTH_TOKEN=${new_gitea_token}
+else
+    # If the token exists, then we must have been passed its value.
+    # If we were not, then we fail hard here.
+    if [[ -z ${CERC_GITEA_AUTH_TOKEN} ]] ; then
+        echo "FATAL error: gitea auth token \"${CERC_GITEA_TOKEN_NAME}\" already exists but no CERC_GITEA_AUTH_TOKEN was provided"
+        exit 1
+    fi
+fi
+# Now that we're sure the token exists and is set in CERC_GITEA_AUTH_TOKEN,
+# we can proceed with token-authenticated API requests below
+# Create org
+# First check if it already exists
+curl -s "${GITEA_URL_PREFIX}/api/v1/admin/users/${GITEA_USER}/orgs" \
+  -H "Authorization: token ${CERC_GITEA_AUTH_TOKEN}" \
+  -H "Content-Type: application/json" \
+  -H  "accept: application/json" \
+  | jq --exit-status -r 'to_entries[] | select(.value.name == "'${GITEA_NEW_ORGANIZATION}'")' > /dev/null
+if [[ $? != 0 ]] ; then
+    # If it doesn't exist, create it
+    # See: https://discourse.gitea.io/t/create-remove-organization-through-api/478
+    curl -s -X POST "${GITEA_URL_PREFIX}/api/v1/admin/users/${GITEA_USER}/orgs" \
+      -H "Authorization: token ${CERC_GITEA_AUTH_TOKEN}" \
+      -H "Content-Type: application/json" \
+      -H  "accept: application/json" \
+      -d '{"username": "'${GITEA_NEW_ORGANIZATION}'"}' > /dev/null
+    echo "Created the organization ${GITEA_NEW_ORGANIZATION}"
+fi
+
+
+# Seed a token for act_runner registration.
+${compose_command} exec db \
+    psql -U gitea -d gitea -c "INSERT INTO public.action_runner_token(token, owner_id, repo_id, is_active, created, updated, deleted) VALUES('${CERC_GITEA_RUNNER_REGISTRATION_TOKEN}', 0, 0, 't', 1679000000, 1679000000, NULL);" >/dev/null
+
+echo "NOTE: Gitea was configured to use host name: gitea.local, ensure that this resolves to localhost, e.g. with sudo vi /etc/hosts"
+if ! [[ -n "$CERC_GITEA_SET_NEW_ADMIN_PASSWORD" ]]; then
+    echo "NOTE: Gitea was configured with admin user and password: ${GITEA_USER}, ${GITEA_PASSWORD}"
+    echo "NOTE: Please make a secure note of the password in order to log in as the admin user"
+fi
+echo "Success, gitea is properly initialized"

gitea/migrate-repo.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Script that calls the Giteap API to migrate one repo from
+# a source hosting platform into that Gitea instance
+
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+
+if ! [[ $# -eq 1 ]]; then
+    echo "Illegal number of parameters" >&2
+    exit 1
+fi
+repo_to_migrate=$1
+
+if [[ -z "${CERC_GITEA_AUTH_TOKEN}" ]]; then
+    echo "CERC_GITEA_AUTH_TOKEN is not set" >&2
+    exit 1
+fi
+if [[ -z "${CERC_GITEA_API_URL}" ]]; then
+    echo "CERC_GITEA_API_URL is not set" >&2
+    exit 1
+fi
+if [[ "${CERC_GITEA_MIRROR_REPO}" == "true" ]]; then
+    is_mirror=true
+else
+    is_mirror=false
+fi
+
+gitea_target_org=$(dirname ${repo_to_migrate})
+gitea_target_repo_name=$(basename ${repo_to_migrate})
+# Sanity check the repo name
+if [[ -z "${gitea_target_org}" ]]; then
+    echo "${repo_to_migrate} is not a valid repo name" >&2
+    exit 1
+fi
+if [[ -z "${gitea_target_repo_name}" ]]; then
+    echo "${repo_to_migrate} is not a valid repo name" >&2
+    exit 1
+fi
+
+github_repo_url="https://github.com/${repo_to_migrate}"
+echo "Migrating repo: ${repo_to_migrate} (mirror:${is_mirror})"
+# Note use: --trace-ascii - \ below to see the raw request
+migrate_response=$( curl -s -X POST "${CERC_GITEA_API_URL}/api/v1/repos/migrate" \
+      -H "Authorization: token ${CERC_GITEA_AUTH_TOKEN}" \
+      -H "Content-Type: application/json" \
+      -H  "accept: application/json" \
+      -d @- << EOF
+{
+    "clone_addr": "${github_repo_url}",
+    "mirror": ${is_mirror},
+    "repo_name": "${gitea_target_repo_name}",
+    "repo_owner": "${gitea_target_org}"
+}
+EOF
+)
+echo Migrated to: $(echo ${migrate_response} | jq -r .html_url)

gitea/run-this-first.sh

@@ -1,4 +1,7 @@
-#!/bin/sh
+#!/usr/bin/env bash
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
 mkdir -p ./gitea
 mkdir -p ./gitea/ssh
-mkdir -p ./postgres
+mkdir -p ./act-runner

tls-proxy/README.md

@@ -0,0 +1,2 @@
+# tls-proxy
+Automated deployment of TLS reverse proxy provisioned with Let's Encrypt certificate

tls-proxy/docker-compose.yml

@@ -0,0 +1,27 @@
+services:
+
+    proxy:
+        image: nginx:stable-bullseye
+        command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g \"daemon off;\"'"
+        ports:
+            - 80:80
+            - 443:443
+        volumes:
+            - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
+            - ./certbot/challenge:/data/certbot-challenge:ro
+            - ./certbot/certificates:/data/certificates:ro
+
+    certbot:
+        image: certbot/certbot:v2.5.0
+        volumes: 
+            - ./certbot/certificates:/etc/letsencrypt
+            - ./certbot/challenge:/data-www-challenge
+        entrypoint: "/bin/sh -c 'sleep 300; trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"
+
+    # Hello-world http container useful for test/debugging the proxy
+    # an actual service would be used for production
+    example-webservice:
+        image: crccheck/hello-world
+        ports:
+          - 8000
+

tls-proxy/initialize-tls-proxy.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+# TODO: get from the caller
+LACONIC_TLS_DOMAIN=example.com
+# When we're called nginx and certbot container are up and running and certbot is sleeping before executing renew
+# So we can now ask certbot to issue our initial cert
+tls_certificate_directory=./certbot/certificates/live/${LACONIC_TLS_DOMAIN}
+rm -rf ${tls_certificate_directory}
+# TODO: pass in email from caller
+# TODO: allow staging/dry-run mode
+docker compose exec certbot \
+    certbot certonly --webroot -w /data-www-challenge \
+    --staging \
+    --email ${EMAIL} \
+    -d ${LACONIC_TLS_DOMAIN} \
+    --rsa-key-size 4096 \
+    --agree-tos \
+    --force-renewal

tls-proxy/nginx-config-template

@@ -0,0 +1,39 @@
+events {
+    worker_connections  1024;
+}
+
+http {
+    server_tokens off;
+    charset utf-8;
+
+    server {
+        listen 80 default_server;
+
+        server_name _;
+
+        location ~ /.well-known/acme-challenge/ {
+            root /data/certbot-challenge;
+        }
+
+        location / {
+            proxy_pass ${LACONIC_ORIGIN_SERVICE_URL};
+        }
+    }
+
+    server {
+        listen 443 ssl http2;
+        ssl_certificate     /data/certificates/live/${LACONIC_TLS_DOMAIN}/fullchain.pem;
+        ssl_certificate_key /data/certificates/live/${LACONIC_TLS_DOMAIN}/privkey.pem;
+        server_name ${LACONIC_TLS_DOMAIN};
+        root /var/www/html;
+        index index.php index.html index.htm;
+
+        location / {
+            proxy_pass ${LACONIC_ORIGIN_SERVICE_URL};
+        }
+
+        location ~ /.well-known/acme-challenge/ {
+            root /data/certbot-challenge;
+        }
+    }
+}

tls-proxy/run-this-first.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+set -e
+mkdir -p ./nginx
+mkdir -p ./certbot/certificates
+mkdir -p ./certbot/challenge
+# TODO: get from the caller
+LACONIC_TLS_DOMAIN=example.com
+LACONIC_ORIGIN_SERVICE_URL=http://example-webservice:8000/
+# Expand the config template into the nginx config file
+cat ./nginx-config-template | sed 's/${LACONIC_TLS_DOMAIN}/'${LACONIC_TLS_DOMAIN}'/' | \
+	sed 's/${LACONIC_ORIGIN_SERVICE_URL}/'${LACONIC_ORIGIN_SERVICE_URL}'/' > ./nginx/nginx.conf
+# Create a self-signed cert so nginx will start without us changing its config between pre and post certbot invocation.
+# Check if we have a cert already
+tls_certificate_directory=./certbot/certificates/live/${LACONIC_TLS_DOMAIN}
+tls_certificate_directory_in_container=/etc/letsencrypt/live/${LACONIC_TLS_DOMAIN}
+tls_certificate_file_name=${tls_certificate_directory}/fullchain.pem
+# TODO: this won't work if there's a delay of more than one day between generating the 
+# self signed cert and starting the certbot enrollment process
+if [[ ! -f ${tls_certificate_file_name} ]] ; then
+    echo "Generating self-signed certificate for ${LACONIC_TLS_DOMAIN}:"
+    mkdir -p ${tls_certificate_directory}
+    docker compose run --rm --entrypoint "\
+        openssl req -x509 -nodes -newkey rsa:4096 -days 1 -keyout '${tls_certificate_directory_in_container}/privkey.pem' \
+        -out '${tls_certificate_directory_in_container}/fullchain.pem' -subj '/CN=${LACONIC_TLS_DOMAIN}'" certbot
+echo
+fi