act-runner/Dockerfile.task-executor

@@ -1,56 +0,0 @@
-FROM ubuntu:22.04
-
-# Set system time zone to prevent the tzdata package from hanging looking for user input
-RUN ln -snf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && echo $CONTAINER_TIMEZONE > /etc/timezone
-
-# Install basic tools
-RUN apt update && apt install -y gpg curl wget apt-transport-https ca-certificates lsb-release build-essential
-
-# Add Docker repo
-RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
-
-ARG NODE_MAJOR=18
-
-# Add NodeJS repo
-# See: https://stackoverflow.com/a/77021599/1701505
-RUN set -uex; \
-    apt-get update; \
-    mkdir -p /etc/apt/keyrings; \
-    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
-     | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg; \
-    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" \
-     > /etc/apt/sources.list.d/nodesource.list; \
-    apt-get update; \
-    apt-get install nodejs -y;
-
-# Install Docker
-RUN apt update && apt install -y docker-ce && rm -rf /var/lib/apt/lists/*
-# Install sudo because some actions projects assume it is present, and it is present in GitHub runners
-RUN apt update && apt install -y sudo
-# Make sure we have some other basic tools that scripts expect.
-RUN apt update && apt install -y wget curl jq
-# Install software-properties-common so we have the add-apt-repository command, used by some actions to add a package repo
-RUN apt update && apt install -y software-properties-common
-
-# Packages and files to support dind functionality see: https://github.com/cruizba/ubuntu-dind
-RUN apt update && apt install -y iptables supervisor
-
-COPY modprobe start-docker.sh entrypoint.sh /usr/local/bin/
-COPY supervisor/ /etc/supervisor/conf.d/
-COPY logger.sh /opt/bash-utils/logger.sh
-COPY cgroup-helper.sh /opt/bash-utils/cgroup-helper.sh
-
-RUN chmod +x /usr/local/bin/start-docker.sh \
-	/usr/local/bin/entrypoint.sh \
-	/usr/local/bin/modprobe
-
-ENV DOCKER_HOST "unix:///var/run/dind.sock"
-
-# This VOLUME directive is required for k3d to work, probably because it needs the directory to exist
-# the volume does not need to be mounted.
-VOLUME /var/lib/docker
-
-ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
-CMD ["bash"]
-

act-runner/cgroup-helper.sh

@@ -1,34 +0,0 @@
-
-# This file needs to be source'ed and the function join_cgroup called, by any script that goes on to run kind
-# This is required due to issues with properly virtualizing the cgroup hierarchy that exist at present in docker
-# See: https://github.com/earthly/earthly/blob/main/buildkitd/dockerd-wrapper.sh#L56
-function configure_cgroup() {
-    if [ -f "/sys/fs/cgroup/cgroup.controllers" ]; then
-        echo >&2 "INFO: detected cgroup v2, configuring nested docker group"
-
-        local cgroup_name="nested-dockerd" # NOTE: has to be the same as the function below (local var to prevent overriding in the caller)
-
-        # move script to separate cgroup, to prevent the root cgroup from becoming threaded (which will prevent systemd images (e.g. kind) from running)
-        mkdir /sys/fs/cgroup/${cgroup_name}
-        echo $$ > /sys/fs/cgroup/${cgroup_name}/cgroup.procs
-
-       # This script is run from inside entrypoint.sh
-       # so we also need to move the parent pid into this new group, which is weird
-       # TODO: we should unwrap this so $$ is all we need to move
-        echo 1 > /sys/fs/cgroup/${cgroup_name}/cgroup.procs
-
-        if [ "$(wc -l < /sys/fs/cgroup/cgroup.procs)" != "0" ]; then
-            echo >&2 "WARNING: processes exist in the root cgroup; this may cause errors during cgroup initialization"
-        fi
-
-        root_cgroup_type="$(cat /sys/fs/cgroup/cgroup.type)"
-        if [ "$root_cgroup_type" != "domain" ]; then
-            echo >&2 "WARNING: expected cgroup type of \"domain\", but got \"$root_cgroup_type\" instead"
-        fi
-    fi
-}
-
-function join_cgroup() {
-    local cgroup_name="nested-dockerd" # NOTE: has to be the same as the function above (local var to prevent overriding in the caller)
-    echo $$ > /sys/fs/cgroup/${cgroup_name}/cgroup.procs
-}

act-runner/docker-compose.yml

@@ -1,23 +0,0 @@
-
-services:
-  runner:
-    image: cerc/act-runner:local
-    restart: always
-    environment:
-      - CONFIG_FILE=/config/act-runner-config.yml
-      # Note: eMdEwIzSo87nBh0UFWZlbp308j6TNWr3WhWxQqIc is a static token we use for convenience in stand-alone deployments. Not secure, obviously.
-      - GITEA_RUNNER_REGISTRATION_TOKEN=${CERC_GITEA_RUNNER_REGISTRATION_TOKEN:-eMdEwIzSo87nBh0UFWZlbp308j6TNWr3WhWxQqIc}
-      - GITEA_INSTANCE_URL=${CERC_GITEA_INSTANCE_URL:-http://gitea.local:3000}
-      - GITEA_RUNNER_LABELS=${CERC_GITEA_RUNNER_LABELS:-ubuntu-latest:docker://cerc/act-runner-task-executor:local,ubuntu-22.04:docker://cerc/act-runner-task-executor:local}
-    extra_hosts:
-      - "gitea.local:host-gateway"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-      - act-runner-data:/data
-      - act-runner-config:/config:ro
-    ports:
-      - 8088
-
-volumes:
-  act-runner-data:
-  act-runner-config:

act-runner/entrypoint.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-# Start docker
-start-docker.sh
-
-# Execute specified command
-"$@"

act-runner/logger.sh

@@ -1,24 +0,0 @@
-#!/bin/sh
-# Logger from this post http://www.cubicrace.com/2016/03/log-tracing-mechnism-for-shell-scripts.html
-
-function INFO(){
-    local function_name="${FUNCNAME[1]}"
-    local msg="$1"
-    timeAndDate=`date`
-    echo "[$timeAndDate] [INFO] [${0}] $msg"
-}
-
-
-function DEBUG(){
-    local function_name="${FUNCNAME[1]}"
-    local msg="$1"
-    timeAndDate=`date`
-    echo "[$timeAndDate] [DEBUG] [${0}] $msg"
-}
-
-function ERROR(){
-    local function_name="${FUNCNAME[1]}"
-    local msg="$1"
-    timeAndDate=`date`
-    echo "[$timeAndDate] [ERROR]  $msg"
-}

act-runner/modprobe

@@ -1,20 +0,0 @@
-#!/bin/sh
-set -eu
-
-# "modprobe" without modprobe
-# https://twitter.com/lucabruno/status/902934379835662336
-
-# this isn't 100% fool-proof, but it'll have a much higher success rate than simply using the "real" modprobe
-
-# Docker often uses "modprobe -va foo bar baz"
-# so we ignore modules that start with "-"
-for module; do
-	if [ "${module#-}" = "$module" ]; then
-		ip link show "$module" || true
-		lsmod | grep "$module" || true
-	fi
-done
-
-# remove /usr/local/... from PATH so we can exec the real modprobe as a last resort
-export PATH='/usr/sbin:/usr/bin:/sbin:/bin'
-exec modprobe "$@"

act-runner/post_start.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
-    set -x
-fi

act-runner/pre_start.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
-    set -x
-fi

act-runner/stack/deploy/commands.py

@@ -1,27 +0,0 @@
-# Copyright © 2023 Vulcanize
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
-
-from pathlib import Path
-from shutil import copy
-
-
-def create(context, extra_args):
-    # Our goal here is just to copy the config file for act
-    deployment_config_dir = context.deployment_dir.joinpath("data",
-                                                            "act-runner-config")
-    command_context = extra_args[2]
-    compose_file = [f for f in command_context.cluster_context.compose_files if "act-runner" in f][0]
-    source_config_file = Path(compose_file).parent.joinpath("config", "act-runner-config.yml")
-    copy(source_config_file, deployment_config_dir)

act-runner/start-docker.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-source /opt/bash-utils/logger.sh
-source /opt/bash-utils/cgroup-helper.sh
-
-function wait_for_process () {
-    local max_time_wait=30
-    local process_name="$1"
-    local waited_sec=0
-    while ! pgrep "$process_name" >/dev/null && ((waited_sec < max_time_wait)); do
-        INFO "Process $process_name is not running yet. Retrying in 1 seconds"
-        INFO "Waited $waited_sec seconds of $max_time_wait seconds"
-        sleep 1
-        ((waited_sec=waited_sec+1))
-        if ((waited_sec >= max_time_wait)); then
-            return 1
-        fi
-    done
-    return 0
-}
-
-# Some payloads (e.g. kind) need systemd to run, which in turn requires forking the cgroup hierarchy
-configure_cgroup
-
-INFO "Starting supervisor"
-/usr/bin/supervisord -n >> /dev/null 2>&1 &
-
-INFO "Waiting for docker to be running"
-wait_for_process dockerd
-if [ $? -ne 0 ]; then
-    ERROR "dockerd is not running after max time"
-    exit 1
-else
-    INFO "dockerd is running"
-fi

act-runner/supervisor/dockerd.conf

@@ -1,6 +0,0 @@
-[program:dockerd]
-command=/usr/bin/dockerd -H %(ENV_DOCKER_HOST)s --userland-proxy=false
-autostart=true
-autorestart=true
-stderr_logfile=/var/log/dockerd.err.log
-stdout_logfile=/var/log/dockerd.out.log

gitea/Dockerfile.task-executor

@@ -0,0 +1,14 @@
+FROM ubuntu:22.04
+
+# Install basic tools
+RUN apt update && apt install -y gpg curl apt-transport-https ca-certificates lsb-release build-essential
+
+# Add Docker repo
+RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+# Add NodeJS repo
+RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash -
+
+# Install Docker and NodeJS packages.
+RUN apt update && apt install -y docker-ce nodejs && rm -rf /var/lib/apt/lists/*

gitea/README.md

@@ -25,8 +25,3 @@ GITEA__log__LEVEL=TRACE
 ```
 to the `server` definition in `docker-compose.yml` and re-start.
 Details on how to setup remote debugging of the gitea server inside its container can be found [here](gitea-debugging.md).
-
-#### Action Runners
-
-A Dockerized action runner is deployed by default for the labels `ubuntu-latest` and `ubuntu-22.04`.  Details on deploying
-additional runners can be found [here](../act-runner/act-runner.md).

gitea/act-runner.md

@@ -1,68 +0,0 @@
-## Deploying Action Runners
-
-IMPORTANT NOTE: you should be aware that anyone with the ability to modify code run under a CI job in the host Gitea (this includes anyone with commit rights; anyone with the ability to modify any dependency; anyone with the ability to modify dependent components such as base container images) CAN POTENTIALLY COMPROMISE (hack, take over, steal data from) the machine hosting a runner. Proceed with caution.
-
-### Releases
-Gitea publishes binary releases of [gitea/act_runner](https://gitea.com/gitea/act_runner/releases) for many platform and architectures, which can be used to deploy new action runners simply.
-
-The following example uses `gitea/act_runner` 0.2.6 to deploy a runner on macOS Ventura 13.3 x64.
-
-### Registration Token
-
-> Note: Runners can be registered globally for an entire Gitea instance, for a specific organization, or for a single repo.  This example registers globally.
-
-Before executing the runner, first obtain a registration token by visiting http://gitea.local:3000/admin/actions/runners, clicking the 'Create new Runner' button, and copying the displayed
-registration token, for example, `FTyMBkcK9ErmD0wm8LfBzfXOUUlQA7dBJF6BB64Z`.
-
-### Runner Registration and Startup
-
-After you have obtained a registration token, download the `gitea/act_runner` release matching your platform and architecture and run it as follows:
-
-```
-# Download latest gitea/act_runner release for your platform.
-$ wget https://gitea.com/gitea/act_runner/releases/download/latest/act_runner-0.2.6-darwin-amd64 && chmod a+x act_runner-0.2.6-darwin-amd64
-
-# Register the runner with the Gitea instance using the token obtained above.
-$ ./act_runner-0.2.6-darwin-amd64 register \
-    --instance http://gitea.local:3000 \
-    --labels 'darwin-latest-amd64:host,darwin-13-amd64:host' \
-    --name 'darwin-amd64-001' \
-    --token "FTyMBkcK9ErmD0wm8LfBzfXOUUlQA7dBJF6BB64Z" \
-    --no-interactive
-
-# Launch it in daemon mode, waiting for jobs.
-$ ./act_runner-0.2.6-darwin-amd64 daemon
-```
-
-### Labels
-
-The most important detail in this example is the label.  For the Ubuntu runner which is deployed automatically with this project, the label `ubuntu-latest:docker://cerc/act-runner-task-executor:local` is
-used, which instructs `gitea/act_runner` that a task which `runs-on: ubuntu-latest` should be executed inside an instance of the `cerc/act-runner-task-executor:local` Docker container.  In this example, the label is `darwin-latest-amd64:host`.  This means that a task which `runs-on: darwin-latest-amd64` will be executed natively on the host machine.  Since there are additional security implications when executing tasks
-on the host, only trusted repositories with strict access controls should be allowed to schedule CI jobs on the runner.
-
-### Example Workflow
-
-This very simple workflow will schedule jobs on both macOS (`darwin-latest-amd64`) and Linux (`ubuntu-latest`) runners.
-
-```
-name: macOS test
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  test-macos:
-    name: "Run on macOS"
-    runs-on: darwin-latest-amd64
-    steps:
-      - name: "uname"
-        run: uname -a
-  test-linux:
-    name: "Run on Ubuntu"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "uname"
-        run: uname -a
-```

gitea/config/act-runner-config.yml

@@ -36,15 +36,13 @@ cache:
   # The host of the cache server.
   # It's not for the address to listen, but the address to connect from job containers.
   # So 0.0.0.0 is a bad choice, leave it empty to detect automatically.
-  host: "gitea.local"
+  host: ""
   # The port of the cache server.
   # 0 means to use a random available port.
-  port: 8088
+  port: 0
 
 container:
   # Whether to use privileged mode or not when launching task containers (privileged mode is required for Docker-in-Docker).
   privileged: true
   # And other options to be used when the container is started (eg, --add-host=my.gitea.url:host-gateway).
   options: --add-host=gitea.local:host-gateway --volume "/var/lib/docker"
-  valid_volumes:
-    - act-runner-shared

gitea/docker-compose.yml

@@ -1,7 +1,7 @@
 
 services:
   server:
-    image: gitea/gitea:1.21
+    image: gitea/gitea:1.20
     environment:
       - USER_UID=1000
       - USER_GID=1000
@@ -19,7 +19,7 @@ services:
     extra_hosts:
       - "gitea.local:host-gateway"
     volumes:
-      - gitea-data:/data
+      - ./gitea:/data
       - /etc/timezone:/etc/timezone:ro
       - /etc/localtime:/etc/localtime:ro
     # TODO: remove fixed host port number
@@ -40,8 +40,19 @@ services:
     entrypoint: bash
     command: -c  'usermod -u ${CERC_HOST_UID:-1000} postgres;groupmod -g ${CERC_HOST_GID:-1000} postgres;exec /usr/local/bin/docker-entrypoint.sh postgres'
     volumes:
-      - postgres-data:/var/lib/postgresql/data
+      - ./postgres:/var/lib/postgresql/data
 
-volumes:
+  runner:
-  gitea-data:
+    image: cerc/act-runner:local
-  postgres-data:
+    restart: always
+    environment:
+      - GITEA_RUNNER_REGISTRATION_TOKEN=eMdEwIzSo87nBh0UFWZlbp308j6TNWr3WhWxQqIc
+      - GITEA_INSTANCE_URL=http://gitea.local:3000
+      - GITEA_RUNNER_LABELS=ubuntu-latest:docker://cerc/act-runner-task-executor:local,ubuntu-22.04:docker://cerc/act-runner-task-executor:local
+      - CONFIG_FILE=/config/act-runner-config.yml
+    extra_hosts:
+      - "gitea.local:host-gateway"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./act-runner:/data
+      - ./config:/config:ro

gitea/initialize-gitea.sh

@@ -1,29 +1,16 @@
 #!/usr/bin/env bash
 # Run this script once after bringing up gitea in docker compose
 # TODO: add a check to detect that gitea has not fully initialized yet (no user relation error)
-
+GITEA_USER=gitea_admin
+GITEA_PASSWORD=admin1234
+GITEA_USER_EMAIL=${GITEA_USER}@example.com
+GITEA_NEW_ORGANIZATION=cerc-io
+GITEA_URL_PREFIX=http://localhost:3000
+CERC_GITEA_TOKEN_NAME=laconic-so-publication-token
+CERC_GITEA_RUNNER_REGISTRATION_TOKEN=eMdEwIzSo87nBh0UFWZlbp308j6TNWr3WhWxQqIc
 if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
     set -x
 fi
-
-secure_password() {
-    # use openssl as the source, because it behaves similarly on both linux and macos
-    # we generate extra bytes so that even if tr deletes some chars we will still have plenty
-    openssl rand -base64 32 | tr -d '\/+=' | head -c 10 && echo
-}
-
-GITEA_USER=${CERC_GITEA_NEW_ADMIN_USERNAME:-"gitea_admin"}
-GITEA_PASSWORD=${CERC_GITEA_SET_NEW_ADMIN_PASSWORD:-"$(secure_password)"}
-GITEA_USER_EMAIL=${CERC_GITEA_SET_NEW_ADMIN_EMAIL:-${GITEA_USER}@example.com}
-GITEA_NEW_ORGANIZATION=${CERC_GITEA_NEW_ORGANIZATION:-"cerc-io"}
-GITEA_URL_PREFIX=http://localhost:3000
-CERC_GITEA_TOKEN_NAME=laconic-so-publication-token
-
-if ! [[ -n "$CERC_GITEA_RUNNER_REGISTRATION_TOKEN" ]]; then
-    echo "Warning: using insecure default runner registration token"
-    CERC_GITEA_RUNNER_REGISTRATION_TOKEN=eMdEwIzSo87nBh0UFWZlbp308j6TNWr3WhWxQqIc
-fi
-
 # Create admin user
 # First check if it already exists
 if [[ -z ${CERC_SO_COMPOSE_PROJECT} ]] ; then
@@ -44,11 +31,6 @@ token_response=$( curl -s "${GITEA_URL_PREFIX}/api/v1/users/${GITEA_USER}/tokens
   -u ${GITEA_USER}:${GITEA_PASSWORD} \
   -H "Content-Type: application/json")
 if [[ -n ${token_response} ]] ; then
-    # Simple check for re-running this script. Ideally we should behave more elegantly.
-    if [[ "${token_response}" == *"password is invalid"* ]]; then
-        echo "Note: admin password is invalid, skipping subsqeuent steps"
-        exit 0
-    fi
     echo ${token_response}  | jq --exit-status -r 'to_entries[] | select(.value.name == "'${CERC_GITEA_TOKEN_NAME}'")'
     if [[ $? == 0 ]] ; then
         token_found=1
@@ -64,8 +46,8 @@ if [[ ${token_found} != 1 ]] ; then
       -H "Content-Type: application/json" \
       -d '{"name":"'${CERC_GITEA_TOKEN_NAME}'", "scopes": [ "read:admin", "write:admin", "read:organization", "write:organization", "read:repository", "write:repository", "read:package", "write:package" ] }' \
       | jq -r .sha1 )
-    echo "NOTE: This is your gitea access token: ${new_gitea_token}. Keep it safe and secure, it can not be fetched again from gitea."
+    echo "This is your gitea access token: ${new_gitea_token}. Keep it safe and secure, it can not be fetched again from gitea."
-    echo "NOTE: To use with laconic-so set this environment variable: export CERC_NPM_AUTH_TOKEN=${new_gitea_token}"
+    echo "To use with laconic-so set this environment variable: export CERC_NPM_AUTH_TOKEN=${new_gitea_token}"
     CERC_GITEA_AUTH_TOKEN=${new_gitea_token}
 else
     # If the token exists, then we must have been passed its value.
@@ -97,12 +79,7 @@ fi
 
 
 # Seed a token for act_runner registration.
-${compose_command} exec db \
+docker compose -p ${CERC_SO_COMPOSE_PROJECT} exec db psql -U gitea -d gitea -c "INSERT INTO public.action_runner_token(token, owner_id, repo_id, is_active, created, updated, deleted) VALUES('${CERC_GITEA_RUNNER_REGISTRATION_TOKEN}', 0, 0, 'f', 1679000000, 1679000000, NULL);" >/dev/null
-    psql -U gitea -d gitea -c "INSERT INTO public.action_runner_token(token, owner_id, repo_id, is_active, created, updated, deleted) VALUES('${CERC_GITEA_RUNNER_REGISTRATION_TOKEN}', 0, 0, 't', 1679000000, 1679000000, NULL);" >/dev/null
 
-echo "NOTE: Gitea was configured to use host name: gitea.local, ensure that this resolves to localhost, e.g. with sudo vi /etc/hosts"
+echo "Gitea was configured to use host name: gitea.local, ensure that this resolves to localhost, e.g. with sudo vi /etc/hosts"
-if ! [[ -n "$CERC_GITEA_SET_NEW_ADMIN_PASSWORD" ]]; then
-    echo "NOTE: Gitea was configured with admin user and password: ${GITEA_USER}, ${GITEA_PASSWORD}"
-    echo "NOTE: Please make a secure note of the password in order to log in as the admin user"
-fi
 echo "Success, gitea is properly initialized"