From ee36e21a9dabf999843e04ec2a99589d0d2ef4cf Mon Sep 17 00:00:00 2001 From: srwadleigh <7832+srwadleigh@users.noreply.github.com> Date: Sat, 27 Apr 2024 01:33:23 +0000 Subject: [PATCH] inital role commit --- LICENSE | 20 +++ README.md | 225 +++++++++++++++++++++++++++ defaults/main.yml | 83 ++++++++++ files/config.sh | 4 + files/firewalld-exmaple-k3s.yml | 68 ++++++++ files/firewalld-exmaple-rke2.yml | 64 ++++++++ files/get-kube-tools.sh | 17 ++ files/token.sh | 38 +++++ handlers/main.yml | 11 ++ meta/main.yml | 38 +++++ tasks/k3s/config.yml | 8 + tasks/k3s/main.yml | 22 +++ tasks/main.yml | 152 ++++++++++++++++++ tasks/rke2/config.yml | 19 +++ tasks/rke2/main.yml | 41 +++++ tasks/rke2/proxy.yml | 50 ++++++ tasks/start.yml | 8 + templates/k3s-config.yaml.j2 | 78 ++++++++++ templates/k3s-kubelet.config.j2 | 5 + templates/rke2-calico-config.yaml.j2 | 13 ++ templates/rke2-canal-config.yaml.j2 | 17 ++ templates/rke2-canal.conf.j2 | 2 + templates/rke2-cilium-config.yaml.j2 | 25 +++ templates/rke2-config.yaml.j2 | 81 ++++++++++ templates/rke2-proxy.j2 | 3 + vars/server-taint.yml | 5 + vars/systems/RedHat.yml | 4 + vars/types/k3s.yml | 45 ++++++ vars/types/rke2.yml | 41 +++++ 29 files changed, 1187 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 defaults/main.yml create mode 100644 files/config.sh create mode 100644 files/firewalld-exmaple-k3s.yml create mode 100644 files/firewalld-exmaple-rke2.yml create mode 100755 files/get-kube-tools.sh create mode 100755 files/token.sh create mode 100644 handlers/main.yml create mode 100644 meta/main.yml create mode 100644 tasks/k3s/config.yml create mode 100644 tasks/k3s/main.yml create mode 100644 tasks/main.yml create mode 100644 tasks/rke2/config.yml create mode 100644 tasks/rke2/main.yml create mode 100644 tasks/rke2/proxy.yml create mode 100644 tasks/start.yml create mode 100644 templates/k3s-config.yaml.j2 create mode 100644 templates/k3s-kubelet.config.j2 create mode 100644 templates/rke2-calico-config.yaml.j2 create mode 100644 templates/rke2-canal-config.yaml.j2 create mode 100644 templates/rke2-canal.conf.j2 create mode 100644 templates/rke2-cilium-config.yaml.j2 create mode 100644 templates/rke2-config.yaml.j2 create mode 100644 templates/rke2-proxy.j2 create mode 100644 vars/server-taint.yml create mode 100644 vars/systems/RedHat.yml create mode 100644 vars/types/k3s.yml create mode 100644 vars/types/rke2.yml diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5addd2c --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2024 Shane Wadleigh + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..e27a3b4 --- /dev/null +++ b/README.md @@ -0,0 +1,225 @@ +# ansible-roles-k8s + + - https://docs.k3s.io/ + - https://docs.rke2.io/ + - https://kube-vip.io/ + - https://github.com/sbstp/kubie + - https://kubernetes.io/docs/tasks/tools/ + +## Requirements + +Install `yq` on the local system, this is required for the kubectl formatting handler which places an updated kubeconfig in the local ~/.kube + +Recommended `kubie` for context management after deployment + +## Cluster Example + +cluster hosts + +``` +[k8s_somecluster] +somecluster_control k8s_node_type=bootstrap +somecluster_agent_smith k8s_node_type=agent k8s_external_ip=x.x.x.x +somecluster_agent_jones k8s_node_type=agent k8s_external_ip=x.x.x.x +``` + +cluster tasks + +``` +- name: Setup k8s server node + hosts: somehost + become: true + roles: + - role: k8s + k8s_type: rke2 + k8s_cluster_name: somecluster + k8s_cluster_url: somecluster.somewhere + k8s_cni_interface: enp1s0 + k8s_selinux: true + + - role: firewalld + firewalld_add: + - name: internal + interfaces: + - enp1s0 + masquerade: true + forward: true + interfaces: + - enp1s0 + services: + - dhcpv6-client + - ssh + - http + - https + ports: + - 6443/tcp # kubernetes API + - 9345/tcp # supervisor API + - 10250/tcp # kubelet metrics + - 2379/tcp # etcd client + - 2380/tcp # etcd peer + - 30000-32767/tcp # NodePort range + - 8472/udp # canal/flannel vxlan + - 9099/tcp # canal health checks + + - name: trusted + sources: + - 10.42.0.0/16 + - 10.43.0.0/16 + + - name: public + masquerade: true + forward: true + interfaces: + - enp7s0 + services: + - http + - https + + firewalld_remove: + - name: public + interfaces: + - enp1s0 + services: + - dhcpv6-client + - ssh +``` + +## Retrieve kube config from an existing cluster + +This task will retrieve and format the kubectl config for an existing cluster, this runs automatically during cluster creation. + +`k8s_cluster_name` sets the cluster context +`k8s_cluster_url` sets the server address + +``` +ansible-playbook -i prod/ site.yml --tags=k8s-get-config --limit=k8s_somecluster +``` + +## Basic Cluster Interaction + +``` +kubie ctx +kubectl get node -o wide +kubectl get pods,svc,ds --all-namespaces +``` + +## Deployment and Removal + +Deploy + +``` +ansible-playbook -i hosts site.yml --tags=firewalld,k8s --limit=somehost +``` + +Remove firewall role + +``` +ansible-playbook -i hosts site.yml --tags=firewalld,k8s --extra-vars "firewall_action=remove" --limit=somehost +``` + +There is a task to completely destroy an existing cluster, this will ask for interactive user confirmation and should be used with caution. + +``` +ansible-playbook -i prod/ site.yml --tags=k8s --extra-vars 'k8s_action=destroy' --limit=some_innocent_cluster +``` + +Manual removal commands + +``` +/usr/local/bin/k3s-uninstall.sh +/usr/local/bin/k3s-agent-uninstall.sh + +/usr/local/bin/rke2-uninstall.sh +/usr/local/bin/rke2-agent-uninstall.sh +``` + +## Managing K3S Services + +servers + +``` +systemctl status k3s.service +journalctl -u k3s.service -f +``` + +agents + +``` +systemctl status k3s-agent.service +journalctl -u k3s-agent -f +``` + +uninstall servers + +``` +/usr/local/bin/k3s-uninstall.sh +``` + +uninstall agents + +``` +/usr/local/bin/k3s-agent-uninstall.sh +``` + +## Managing RKE2 Services + +servers + +``` +systemctl status rke2-server.service +journalctl -u rke2-server -f +``` + +agents + +``` +systemctl status rke2-agent.service +journalctl -u rke2-agent -f +``` + +uninstall servers + +``` +/usr/bin/rke2-uninstall.sh +``` + +uninstall agents + +``` +/usr/local/bin/rke2-uninstall.sh +``` + + +override default cannal options + +``` +# /var/lib/rancher/rke2/server/manifests/rke2-canal-config.yaml +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: rke2-canal + namespace: kube-system +spec: + valuesContent: |- + flannel: + iface: "eth1" +``` + +Enable flannels wireguard support under canal + +`kubectl rollout restart ds rke2-canal -n kube-system` + +``` +# /var/lib/rancher/rke2/server/manifests/rke2-canal-config.yaml +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: rke2-canal + namespace: kube-system +spec: + valuesContent: |- + flannel: + backend: "wireguard" +``` diff --git a/defaults/main.yml b/defaults/main.yml new file mode 100644 index 0000000..74f8f5f --- /dev/null +++ b/defaults/main.yml @@ -0,0 +1,83 @@ +--- +# this toggle provides a dangerous way to quickly destroy an entire cluster +# ansible-playbook -i prod/ site.yml --tags=k8s --extra-vars 'k8s_action=destroy' --limit=k3s_innocent_cluster +# create | destroy +k8s_action: create + +# k3s | rke2 +k8s_type: k3s +k8s_channel: stable + +k8s_cluster_name: default +k8s_cluster_url: localhost + +# bootstrap | server | agent +k8s_node_type: bootstrap +k8s_node_ip: "{{ ansible_host }}" + +# sysctl set fs.inotify.max_user_instances +k8s_inotify_max: 1024 + +# hardcoded kublet default value is 110 +k8s_pod_limit: 110 + +# we can set this by platform later +k8s_selinux: false + +# if the host is using network manager, see vars/sys/ for overrides +k8s_has_nm: false + +# if the host is using an http proxy +k8s_http_proxy: false + +# cni +# k8s_cni_type: + +k8s_config_mode: 600 +k8s_api_port: 6443 +k8s_debug: false +k8s_skip_start: false +k8s_taint_servers: false +k8s_flannel_wireguard: false +k8s_disable_kube_proxy: false + +# paths +k8s_install_script: /usr/local/bin/{{ k8s_type }}-install.sh +k8s_config_path: "/etc/rancher/{{ k8s_type }}" +k8s_cmd_path: /usr/local/bin +k8s_nm_path: /etc/NetworkManager/conf.d + +k8s_manifests_path: "/var/lib/rancher/{{ k8s_type }}/server/manifests/" + +# if defined, install manifests +# k8s_manifests: +# - name: cert-manager +# path: https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml + +# Override +# k8s_cluster_name +# k8s_cluster_url + +# Define +# k8s_cluster_token +# you can pre-generate this ina vault with the token.sh script + +# k8s_node_taints +# --node-taint CriticalAddonsOnly=true:NoExecute +# k8s_node_taints: +# - name: CriticalAddonsOnly +# value: true +# effect: NoExecute + +# these are provided simply for the opportunity to override in cases where some ajustment isnt supported by the config templates +# k8s_install_bootstrap: >- +# server --cluster-init --tls-san {{ k8s_cluster_url }} --node-taint CriticalAddonsOnly=true:NoExecute +# {% if k8s_disable is defined %} +# {% for disable in k8s_disable %} +# --disable={{ disable }} +# {% endfor %} +# {% endif %} + +# k8s_install_agent: >- +# agent --kubelet-arg=config=/etc/rancher/k3s/kubelet.config --node-ip={{ ansible_host }} +# {% if k8s_external_ip is defined %}--node-external-ip={{ k8s_external_ip }}{% endif %} diff --git a/files/config.sh b/files/config.sh new file mode 100644 index 0000000..49bc187 --- /dev/null +++ b/files/config.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +yq e '.contexts[].name = "$K3S_CONTEXT"' -i config + diff --git a/files/firewalld-exmaple-k3s.yml b/files/firewalld-exmaple-k3s.yml new file mode 100644 index 0000000..8a0c47f --- /dev/null +++ b/files/firewalld-exmaple-k3s.yml @@ -0,0 +1,68 @@ +--- +firewalld_add: + - name: internal + masquerade: false + forward: true + interfaces: + - eth0 + services: + - dhcpv6-client + - ssh + - http + - https + ports: + - 9100/tcp # node exporter + - 6443/tcp # kubernetes API + - 9345/tcp # supervisor API + - 10250/tcp # kubelet metrics + - 2379/tcp # etcd client + - 2380/tcp # etcd peer + - 30000-32767/tcp # NodePort range + + # Spegel + - 5001/tcp # embedded distributed registry + + # Flannel CNI + - 8472/udp # flannel vxlan + - 51820/udp # wireguard ipv4 + - 51821/udp # wireguard ipv6 + + # Canal CNI + # - 8472/udp # canal vxlan + # - 9099/tcp # canal health checks + # - 51820/udp # canal WireGuard IPv4 + # - 51821/udp # canal WireGuard IPv6/dual-stack + + # Cilium CNI + #- 8472/udp # cilium vxlan + #- 4240/tcp # cilium health checks + #- 8/0/icmp # cilium health checks + + #- 51871/udp # cilium wireguard + #- 4244/tcp # hubble relay + #- 4245/tcp # hubble relay + #- 9962/tcp # cilium agent prometheus + #- 9963/tcp # cilium operator prometheus + #- 9964/tcp # cilium proxy prometheus + #- 2379-2380/tcp # etcd access + + # Calico CNI + # - 179/tcp # calico bgp + # - 4789/udp # calico vxlan + # - 5473/tcp # calico typha + # - 9098/tcp # calico typha health checks + # - 9099/tcp # calico health checks + # - 51820/udp # calico WireGuard IPv4 + # - 51821/udp # calico WireGuard IPv6/dual-stack + + - name: trusted + sources: + - 10.42.0.0/16 + - 10.43.0.0/16 + - 10.0.0.0/16 + +firewalld_remove: + - name: public + services: + - dhcpv6-client + - ssh diff --git a/files/firewalld-exmaple-rke2.yml b/files/firewalld-exmaple-rke2.yml new file mode 100644 index 0000000..12248fe --- /dev/null +++ b/files/firewalld-exmaple-rke2.yml @@ -0,0 +1,64 @@ +--- +firewalld_add: + - name: internal + masquerade: false + forward: true + interfaces: + - eth0 + services: + - dhcpv6-client + - ssh + - http + - https + ports: + - 9100/tcp # node exporter + - 6443/tcp # kubernetes API + - 9345/tcp # supervisor API + - 10250/tcp # kubelet metrics + - 2379/tcp # etcd client + - 2380/tcp # etcd peer + - 2381/tcp # etcd metrics + - 30000-32767/tcp # NodePort range + + # Canal CNI - Default - + - 8472/udp # canal vxlan + - 9099/tcp # canal health checks + - 51820/udp # canal WireGuard IPv4 + - 51821/udp # canal WireGuard IPv6/dual-stack + + # Cilium CNI + #- 8472/udp # cilium vxlan + #- 4240/tcp # cilium health checks + #- 8/0/icmp # cilium health checks + #- 51871/udp # cilium wireguard + #- 4244/tcp # hubble relay + #- 4245/tcp # hubble relay + #- 9962/tcp # cilium agent prometheus + #- 9963/tcp # cilium operator prometheus + #- 9964/tcp # cilium proxy prometheus + #- 2379-2380/tcp # etcd access + + # Calico CNI + # - 179/tcp # calico bgp + # - 4789/udp # calico vxlan + # - 5473/tcp # calico typha + # - 9098/tcp # calico typha health checks + # - 9099/tcp # calico health checks + # - 51820/udp # calico WireGuard IPv4 + # - 51821/udp # calico WireGuard IPv6/dual-stack + + # Flannel CNI + #- 8472/udp # flannel vxlan + #- 4789/udp + + - name: trusted + sources: + - 10.42.0.0/16 + - 10.43.0.0/16 + - 10.0.0.0/16 + +firewalld_remove: + - name: public + services: + - dhcpv6-client + - ssh diff --git a/files/get-kube-tools.sh b/files/get-kube-tools.sh new file mode 100755 index 0000000..1ca066e --- /dev/null +++ b/files/get-kube-tools.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +INSTALL_PATH="/usr/local/bin/" +INSTALL_ARCH="amd64" +KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt) +KUBIE_VERSION="latest" +YQ_VERSION="latest" + +wget -qO ${INSTALL_PATH}/kubectl https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${INSTALL_ARCH}/kubectl +chmod a+x ${INSTALL_PATH}/kubectl + +wget -qO ${INSTALL_PATH}/kubie https://github.com/sbstp/kubie/releases/${KUBIE_VERSION}/download/kubie-linux-${INSTALL_ARCH} +chmod a+x ${INSTALL_PATH}/kubie + +wget -qO ${INSTALL_PATH}/yq https://github.com/mikefarah/yq/releases/${YQ_VERSION}/download/yq_linux_${INSTALL_ARCH} +chmod a+x ${INSTALL_PATH}/yq + diff --git a/files/token.sh b/files/token.sh new file mode 100755 index 0000000..86bca3f --- /dev/null +++ b/files/token.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +vault_output="$1" +vault_regex=".*\.yml$" +vault_var_name="k8s_cluster_token" +token="$(openssl rand -hex 16)" + +print_token() { + echo "$token" +} + +print_yaml() { + printf -- "---\n$vault_var_name: %s\n" "$token" +} + +encrypt_token() { + ansible-vault encrypt_string "$token" --name "$vault_var_name" +} + +encrypt_yaml() { + print_yaml | ansible-vault encrypt +} + +if [ -n "$vault_output" ]; then + if [[ $vault_output =~ $vault_regex ]]; then + if [ -f "$vault_output" ]; then + echo "output file already exists, no token generated" + exit 0 + else + encrypt_yaml > "$vault_output" + fi + else + echo "supplied output file should end with .yml" + exit 1 + fi +else + encrypt_token +fi diff --git a/handlers/main.yml b/handlers/main.yml new file mode 100644 index 0000000..9072a06 --- /dev/null +++ b/handlers/main.yml @@ -0,0 +1,11 @@ +--- + +- name: Update k8s Local Config + listen: "k8s-update-local-config" + delegate_to: localhost + become: false + ansible.builtin.shell: | + yq e '.clusters[].name = "{{ k8s_cluster_name }}"' -i ~/.kube/config-{{ k8s_cluster_name }}.yaml + yq e '.contexts[].name = "{{ k8s_cluster_context | d(k8s_cluster_name) }}"' -i ~/.kube/config-{{ k8s_cluster_name }}.yaml + yq e '(.clusters[] | select(.name == "{{ k8s_cluster_name }}")).cluster.server = "https://{{ k8s_cluster_url }}:{{ k8s_api_port }}"' -i ~/.kube/config-{{ k8s_cluster_name }}.yaml + yq e '(.contexts[] | select(.name == "{{ k8s_cluster_name }}")).context.cluster = "{{ k8s_cluster_name }}"' -i ~/.kube/config-{{ k8s_cluster_name }}.yaml diff --git a/meta/main.yml b/meta/main.yml new file mode 100644 index 0000000..c255138 --- /dev/null +++ b/meta/main.yml @@ -0,0 +1,38 @@ +--- +dependencies: [] + +galaxy_info: + role_name: k8s + author: srw + description: Ansible role for configuring k3s and rke2 kubernetes clusters + company: "NMD, LLC" + license: "license (BSD, MIT)" + min_ansible_version: "2.10" + platforms: + - name: Fedora + versions: + - all + - name: Debian + versions: + - buster + - bullseye + - bookworm + - name: Ubuntu + versions: + - bionic + - focal + - jammy + - name: Alpine + version: + - all + - name: ArchLinux + versions: + - all + galaxy_tags: + - server + - system + - containers + - kubernetes + - k8s + - k3s + - rke2 diff --git a/tasks/k3s/config.yml b/tasks/k3s/config.yml new file mode 100644 index 0000000..350e4ce --- /dev/null +++ b/tasks/k3s/config.yml @@ -0,0 +1,8 @@ +--- + + # PRE-DEPLOY + - name: template k3s kubelet config + ansible.builtin.template: + src: "templates/k3s-kubelet.config.j2" + dest: "/etc/rancher/k3s/kubelet.config" + mode: 0644 diff --git a/tasks/k3s/main.yml b/tasks/k3s/main.yml new file mode 100644 index 0000000..edfd815 --- /dev/null +++ b/tasks/k3s/main.yml @@ -0,0 +1,22 @@ +--- + + # BOOTSTRAP + - name: k3s boostrap initial server node + ansible.builtin.shell: "{{ k8s_install_script }}" + environment: "{{ k8s_env | combine({'INSTALL_K3S_EXEC': '{{ k8s_install_bootstrap }}'}) }}" + when: + - k8s_node_type == "bootstrap" + + # ADD SERVERS + - name: k3s add additional server nodes + ansible.builtin.shell: "{{ k8s_install_script }}" + environment: "{{ k8s_env | combine({'INSTALL_K3S_EXEC': '{{ k8s_install_server }}'}) }}" + when: + - k8s_node_type == "server" + + # ADD AGENTS + - name: k3s add agent nodes + ansible.builtin.shell: "{{ k8s_install_script }}" + environment: "{{ k8s_env | combine({'INSTALL_K3S_EXEC': '{{ k8s_install_agent }}'}) }}" + when: + - k8s_node_type == "agent" diff --git a/tasks/main.yml b/tasks/main.yml new file mode 100644 index 0000000..5028c07 --- /dev/null +++ b/tasks/main.yml @@ -0,0 +1,152 @@ +--- +- name: Setup Environment + tags: + - k8s + - k8s-config + block: + + - name: gather local facts + set_fact: + local_user: "{{ lookup('env', 'USER') }}" + delegate_to: localhost + + # useful to set this bc k8s_node_type has a special value bootstrap which is not an actual type + - name: set true node type + set_fact: + node_type: "{{ 'agent' if k8s_node_type == 'agent' else 'server' }}" + + - name: load type specific values + ansible.builtin.include_vars: + file: "types/{{ k8s_type }}.yml" + + - name: load system specific values + ansible.builtin.include_vars: "{{ item }}" + with_first_found: + - files: + - "systems/{{ ansible_os_family }}-{{ ansible_distribution_major_version }}.yml" + - "systems/{{ ansible_os_family }}.yml" + - "systems/{{ ansible_distribution }}.yml" + - "systems/{{ ansible_system }}.yml" + skip: true + +# +# CREATE CLUSTER +# +- name: Cluster Creation + tags: k8s + block: + + - name: load server node taints + ansible.builtin.include_vars: + file: "server-taint.yml" + when: + - k8s_taint_servers and k8s_node_type != "agent" + + - name: increase open file limit + ansible.posix.sysctl: + name: fs.inotify.max_user_instances + value: "{{ k8s_inotify_max }}" + state: present + + - name: download install script + ansible.builtin.get_url: + url: "{{ k8s_install_url }}" + timeout: 120 + dest: "{{ k8s_install_script }}" + owner: root + group: root + mode: 0755 + + # CLUSTER CONFIG + - name: check config paths + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: 0755 + loop: + - "{{ k8s_config_path }}" + - "{{ k8s_manifests_path }}" + tags: + - k8s-config + + - name: template cluster config + ansible.builtin.template: + src: "templates/{{ k8s_type }}-config.yaml.j2" + dest: "{{ k8s_config_path }}/config.yaml" + mode: 0600 + tags: + - k8s-config + + - name: type specific configuration + ansible.builtin.include_tasks: "{{ k8s_type }}/config.yml" + tags: + - k8s-config + + # DEPLOY CLUSTER + - name: beging cluster creation + ansible.builtin.include_tasks: "{{ k8s_type }}/main.yml" + + # KUBE CONFIG + - name: fetch kube config + ansible.builtin.fetch: + src: "{{ k8s_config_path }}/{{ k8s_type }}.yaml" + dest: "~/.kube/config-{{ k8s_cluster_name }}.yaml" + flat: yes + notify: + - k8s-update-local-config + when: + - k8s_node_type == "bootstrap" + tags: + - k8s-get-kubeconf + + # DEPLOY MANIFESTS + + # END Cluster Creation + when: + - k8s_action == "create" + +# +# DESTORY CLUSTER +# +# this is very dangerous and should be handled with care when not actively testing with disposable cluster iterations +- name: Destroy K8s cluster + tags: k8s + block: + + - name: confirm cluster destruction + delegate_to: localhost + run_once: true + become: false + pause: + prompt: "=== WARNING === Are you sure you want to DESTROY the cluster: {{ k8s_cluster_name | string | upper }}? (yes/no)" + register: destroy_confirmation + + - name: set confirmation fact + set_fact: + cluster_destruction: "{{ destroy_confirmation.user_input }}" + + - name: delete cluster config + delegate_to: localhost + run_once: true + become: false + file: + path: "~/.kube/config-{{ k8s_cluster_name }}.yaml" + state: absent + when: + - cluster_destruction + + - name: destroy nodes + ansible.builtin.shell: "{{ k8s_cmd_path }}/{{ k8s_type }}-uninstall.sh" + when: + - k8s_node_type != "agent" or k8s_type == "rke2" + - cluster_destruction + + - name: destroy k3s agent nodes + ansible.builtin.shell: "{{ k8s_cmd_path }}/{{ k8s_type }}-agent-uninstall.sh" + when: + - k8s_node_type == "agent" and k8s_type == "k3s" + - cluster_destruction + + # END Cluster Destruction + when: + - k8s_action == "destroy" diff --git a/tasks/rke2/config.yml b/tasks/rke2/config.yml new file mode 100644 index 0000000..59695c0 --- /dev/null +++ b/tasks/rke2/config.yml @@ -0,0 +1,19 @@ +--- + + # HTTP PROXY + - name: http proxy tasks + ansible.builtin.include_tasks: "{{ k8s_type }}/proxy.yml" + tags: + - k8s-config + + # CANAL NM CONFIG + - name: template nm canal config + ansible.builtin.template: + src: "templates/{{ k8s_type }}-canal.conf.j2" + dest: "{{ k8s_nm_path }}/{{ k8s_type }}-canal.conf" + mode: 0600 + when: + - k8s_cni_type == "canal" + - k8s_has_nm + tags: + - k8s-config diff --git a/tasks/rke2/main.yml b/tasks/rke2/main.yml new file mode 100644 index 0000000..9931c52 --- /dev/null +++ b/tasks/rke2/main.yml @@ -0,0 +1,41 @@ +--- + + # BOOTSTRAP + - name: rke2 boostrap initial server node + ansible.builtin.shell: "{{ k8s_install_script }}" + environment: "{{ k8s_env | combine({'INSTALL_RKE2_TYPE': 'server'}) }}" + when: + - k8s_node_type == "bootstrap" + + - name: rke2 template cni manifests + ansible.builtin.template: + src: "templates/{{ k8s_type }}-{{ k8s_cni_type }}-config.yaml.j2" + dest: "{{ k8s_manifests_path }}/{{ k8s_type }}-{{ k8s_cni_type }}-config.yaml" + mode: 0600 + when: + - k8s_node_type == "bootstrap" + + - name: rke2 start bootstrap node + ansible.builtin.include_tasks: start.yml + when: + - k8s_node_type == "bootstrap" + + # ADD SERVERS + - name: rke2 add additional server nodes + ansible.builtin.shell: "{{ k8s_install_script }}" + environment: "{{ k8s_env | combine({'INSTALL_RKE2_TYPE': 'server'}) }}" + when: + - k8s_node_type == "server" + + # ADD AGENTS + - name: rke2 add agent nodes + ansible.builtin.shell: "{{ k8s_install_script }}" + environment: "{{ k8s_env | combine({'INSTALL_RKE2_TYPE': 'agent'}) }}" + when: + - k8s_node_type == "agent" + + # POST-DEPLOY + - name: rke2 start additional nodes + ansible.builtin.include_tasks: start.yml + when: + - k8s_node_type != "bootstrap" diff --git a/tasks/rke2/proxy.yml b/tasks/rke2/proxy.yml new file mode 100644 index 0000000..9f49d6b --- /dev/null +++ b/tasks/rke2/proxy.yml @@ -0,0 +1,50 @@ +--- +- name: http proxy detection and setup + tags: + - k8s + - k8s-config + block: + + - name: check for existing http_proxy + shell: echo $http_proxy + register: http_proxy + ignore_errors: true + changed_when: false + + - name: check for existing https_proxy + shell: echo $https_proxy + register: https_proxy + ignore_errors: true + changed_when: false + + - name: check for existing no_proxy + shell: echo $no_proxy + register: no_proxy + ignore_errors: true + changed_when: false + + - name: Set fact for HTTP_PROXY + set_fact: + k8s_http_proxy: "{{ http_proxy.stdout | default('') }}" + when: + - http_proxy.stdout != "" + + - name: Set fact for HTTPS_PROXY + set_fact: + k8s_https_proxy: "{{ https_proxy.stdout | default('') }}" + when: + - https_proxy.stdout != "" + + - name: Set fact for NO_PROXY + set_fact: + k8s_no_proxy: "{{ no_proxy.stdout | default('') }}" + when: no_proxy.stdout != "" + + - name: template rke2 http proxy + ansible.builtin.template: + src: "templates/{{ k8s_type }}-proxy.j2" + dest: "/etc/default/rke2-{{ node_type }}" + mode: 0644 + when: + - http_proxy.stdout != "" + - https_proxy.stdout != "" diff --git a/tasks/start.yml b/tasks/start.yml new file mode 100644 index 0000000..e404afa --- /dev/null +++ b/tasks/start.yml @@ -0,0 +1,8 @@ +--- + +# handlers dont execute in time so we include this as a task +- name: enable "{{ k8s_type }}" service + ansible.builtin.systemd: + name: "{{ k8s_type }}-{{ node_type }}" + state: restarted + enabled: true diff --git a/templates/k3s-config.yaml.j2 b/templates/k3s-config.yaml.j2 new file mode 100644 index 0000000..de9b6c2 --- /dev/null +++ b/templates/k3s-config.yaml.j2 @@ -0,0 +1,78 @@ +# template generated via ansible by {{ local_user }} at {{ ansible_date_time.date }} {{ ansible_date_time.time }} + +token: {{ k8s_cluster_token }} +{% if k8s_cluster_url is defined and k8s_node_type != "bootstrap" -%} +server: https://{{ k8s_cluster_url }}:{{ k8s_api_port }} +{% endif -%} +{% if k8s_node_type == "bootstrap" -%} +cluster-init: true +{% endif -%} +debug: {{ k8s_debug | string | lower }} +{% if k8s_node_type != "agent" -%} +write-kubeconfig-mode: {{ k8s_config_mode }} +{% if k8s_tls_san is defined and k8s_node_type != "agent" -%} +{% for san in k8s_tls_san -%} +tls-san: + - "{{ san }}" +{% endfor -%} +{% elif k8s_cluster_url is defined and k8s_node_type != "agent" -%} +tls-san: {{ k8s_cluster_url }} +{% endif %} +{% if k8s_selinux -%} +selinux: true +{% endif -%} + +{% if k8s_disable is defined and k8s_node_type != "agent" %} +# disable builtin services +{% for disable in k8s_disable %} +disable: {{ disable }} +{% endfor -%} +{% endif -%} +{% endif %} + +# node network +{% if k8s_node_ip is defined -%} +node-ip: {{ k8s_node_ip }} +{% endif -%} +{% if k8s_external_ip is defined -%} +node-external-ip: {{ k8s_external_ip }} +{% endif -%} + +{% if k8s_flannel_backend is defined and k8s_node_type != "agent" -%} +# cofigure or disable flannel cni +flannel-backend: {{ k8s_flannel_backend }} +flannel-ipv6-masq: {{ k8s_flannel_ipv6_masq }} +flannel-external-ip: {{ k8s_flannel_external_ip }} +{% endif %} + +{% if k8s_node_taints is defined -%} +# initial node taints +{% for taint in k8s_node_taints -%} +node-taint: + - "{{ taint.name }}={{ taint.value }}:{{ taint.effect }}" +{% endfor -%} +{% endif %} + +{% if k8s_node_lables is defined -%} +# initial node labels +{% for label in k8s_node_lables -%} +node-label: + - "{{ label.name }}={{ label.value }}" +{% endfor -%} +{% endif %} + +{% if k8s_kubelet_args is defined %} +# kubelet configuration +{% for kublet_arg in k8s_kubelet_args %} +kubelet-arg: + - "{{ kublet_arg }}" + +{% endfor -%} +{% endif %} + +{% if k8s_additional_configs is defined %} +{% for k8s_config in k8s_additional_configs %} +{{ k8s_config.name }}: + - "{{ k8s_config.value }}" +{% endfor -%} +{% endif %} diff --git a/templates/k3s-kubelet.config.j2 b/templates/k3s-kubelet.config.j2 new file mode 100644 index 0000000..ebe153e --- /dev/null +++ b/templates/k3s-kubelet.config.j2 @@ -0,0 +1,5 @@ +# template generated via ansible by {{ local_user }} at {{ ansible_date_time.date }} {{ ansible_date_time.time }} + +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +maxPods: {{ k8s_pod_limit }} diff --git a/templates/rke2-calico-config.yaml.j2 b/templates/rke2-calico-config.yaml.j2 new file mode 100644 index 0000000..fb7acd9 --- /dev/null +++ b/templates/rke2-calico-config.yaml.j2 @@ -0,0 +1,13 @@ +# template generated via ansible by {{ local_user }} at {{ ansible_date_time.date }} {{ ansible_date_time.time }} +# /var/lib/rancher/rke2/server/manifests/rke2-calico-config.yaml +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: rke2-calico + namespace: kube-system +spec: + valuesContent: |- + installation: + calicoNetwork: + mtu: 9000 diff --git a/templates/rke2-canal-config.yaml.j2 b/templates/rke2-canal-config.yaml.j2 new file mode 100644 index 0000000..36c2d74 --- /dev/null +++ b/templates/rke2-canal-config.yaml.j2 @@ -0,0 +1,17 @@ + +# template generated via ansible by {{ local_user }} at {{ ansible_date_time.date }} {{ ansible_date_time.time }} +# /var/lib/rancher/rke2/server/manifests/rke2-canal-config.yaml +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: rke2-canal + namespace: kube-system +spec: + valuesContent: |- + flannel: +{% if k8s_flannel_wireguard %} + backend: "wireguard" +{% else %} + iface: "{{ k8s_cni_interface }}" +{% endif %} diff --git a/templates/rke2-canal.conf.j2 b/templates/rke2-canal.conf.j2 new file mode 100644 index 0000000..16476e2 --- /dev/null +++ b/templates/rke2-canal.conf.j2 @@ -0,0 +1,2 @@ +[keyfile] +unmanaged-devices=interface-name:cali*;interface-name:flannel* diff --git a/templates/rke2-cilium-config.yaml.j2 b/templates/rke2-cilium-config.yaml.j2 new file mode 100644 index 0000000..bca3484 --- /dev/null +++ b/templates/rke2-cilium-config.yaml.j2 @@ -0,0 +1,25 @@ +# template generated via ansible by {{ local_user }} at {{ ansible_date_time.date }} {{ ansible_date_time.time }} +# /var/lib/rancher/rke2/server/manifests/rke2-cilium-config.yaml +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: rke2-cilium + namespace: kube-system +spec: + valuesContent: |- + eni: + enabled: true +{% if k8s_disable_kube_proxy %} + kubeProxyReplacement: true + k8sServiceHost: {{ k8s_cluster_url }} + k8sServicePort: {{ k8s_api_port }} +{% endif %} +{% if k8s_cilium_hubble %} + hubble: + enabled: true + relay: + enabled: true + ui: + enabled: true +{% endif %} diff --git a/templates/rke2-config.yaml.j2 b/templates/rke2-config.yaml.j2 new file mode 100644 index 0000000..2dff4f5 --- /dev/null +++ b/templates/rke2-config.yaml.j2 @@ -0,0 +1,81 @@ +# template generated via ansible by {{ local_user }} at {{ ansible_date_time.date }} {{ ansible_date_time.time }} + +token: {{ k8s_cluster_token }} +{% if k8s_cluster_url is defined and k8s_node_type != "bootstrap" -%} +server: https://{{ k8s_cluster_url }}:{{ k8s_supervisor_port }} +{% endif -%} +debug: {{ k8s_debug | string | lower }} +{% if k8s_node_type != "agent" -%} +write-kubeconfig-mode: {{ k8s_config_mode }} +{% if k8s_tls_san is defined and k8s_node_type != "agent" -%} +{% for san in k8s_tls_san -%} +tls-san: + - "{{ san }}" +{% endfor -%} +{% elif k8s_cluster_url is defined and k8s_node_type != "agent" -%} +tls-san: {{ k8s_cluster_url }} +{% endif %} +{% if k8s_selinux -%} +selinux: true +{% endif -%} + +{% if k8s_cni_type is defined -%} +cni: {{ k8s_cni_type }} +{% endif -%} +{% if k8s_disable_kube_proxy %} +disable-kube-proxy: true +{% endif -%} +{% if k8s_disable is defined and k8s_node_type != "agent" %} +# disable builtin services +{% for disable in k8s_disable %} +disable: {{ disable }} +{% endfor -%} +{% endif -%} +{% endif %} + +# node network +{% if k8s_node_ip is defined -%} +node-ip: {{ k8s_node_ip }} +{% endif -%} +{% if k8s_external_ip is defined -%} +node-external-ip: {{ k8s_external_ip }} +{% endif -%} + +{% if k8s_flannel_backend is defined and k8s_node_type != "agent" -%} +# cofigure or disable flannel cni +flannel-backend: {{ k8s_flannel_backend }} +flannel-ipv6-masq: {{ k8s_flannel_ipv6_masq }} +flannel-external-ip: {{ k8s_flannel_external_ip }} +{% endif %} + +{% if k8s_node_taints is defined -%} +# initial node taints +{% for taint in k8s_node_taints -%} +node-taint: + - "{{ taint.name }}={{ taint.value }}:{{ taint.effect }}" +{% endfor -%} +{% endif %} + +{% if k8s_node_lables is defined -%} +# initial node labels +{% for label in k8s_node_lables -%} +node-label: + - "{{ label.name }}={{ label.value }}" +{% endfor -%} +{% endif %} + +{% if k8s_kubelet_args is defined %} +# kubelet configuration +{% for kublet_arg in k8s_kubelet_args %} +kubelet-arg: + - "{{ kublet_arg }}" + +{% endfor -%} +{% endif %} + +{% if k8s_additional_configs is defined %} +{% for k8s_config in k8s_additional_configs %} +{{ k8s_config.name }}: + - "{{ k8s_config.value }}" +{% endfor -%} +{% endif %} diff --git a/templates/rke2-proxy.j2 b/templates/rke2-proxy.j2 new file mode 100644 index 0000000..079be90 --- /dev/null +++ b/templates/rke2-proxy.j2 @@ -0,0 +1,3 @@ +HTTP_PROXY={{ k8s_http_proxy | d() }} +HTTPS_PROXY={{ k8s_https_proxy | d() }} +NO_PROXY={{ k8s_no_proxy | d() }} \ No newline at end of file diff --git a/vars/server-taint.yml b/vars/server-taint.yml new file mode 100644 index 0000000..f9837ab --- /dev/null +++ b/vars/server-taint.yml @@ -0,0 +1,5 @@ +--- +k8s_node_taints: + - name: CriticalAddonsOnly + value: true + effect: NoExecute diff --git a/vars/systems/RedHat.yml b/vars/systems/RedHat.yml new file mode 100644 index 0000000..11ce5f8 --- /dev/null +++ b/vars/systems/RedHat.yml @@ -0,0 +1,4 @@ +--- +k8s_selinux: true +k8s_has_nm: true +k8s_cmd_path: /usr/bin diff --git a/vars/types/k3s.yml b/vars/types/k3s.yml new file mode 100644 index 0000000..e1f59c6 --- /dev/null +++ b/vars/types/k3s.yml @@ -0,0 +1,45 @@ +--- +# See https://docs.k3s.io/ + +# define k8s_version to deploy a specific version +# channel: stable, latest, testing +k8s_install_url: https://get.k3s.io +k8s_channel_url: https://update.k3s.io/v1-release/channels + +# cluster network (cni) +# flannel-backend: 'vxlan', 'host-gw', 'wireguard-native', 'none' +k8s_flannel_backend: vxlan +k8s_flannel_ipv6_masq: false +k8s_flannel_external_ip: false + +# disable builtin services +k8s_disable: + - "traefik" + +# kubelet configs +# - "kube-reserved=cpu=500m,memory=1Gi,ephemeral-storage=2Gi" +# - "system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=2Gi" +# - "eviction-hard=memory.available<500Mi,nodefs.available<10%" +k8s_kubelet_args: + - config=/etc/rancher/k3s/kubelet.config + +k8s_env: + INSTALL_K3S_CHANNEL_URL: "{{ k8s_channel_url }}" + INSTALL_K3S_CHANNEL: "{{ k8s_channel }}" + INSTALL_K3S_SKIP_START: "{{ k8s_skip_start }}" + + # will attempt to download from channel if not specified + INSTALL_K3S_VERSION: "{{ k8s_version | d() }}" + + # there is some consider for where the token lives after initial node creation, this could get pruned from env or config most likely + #K3S_TOKEN: "{{ k8s_cluster_token }}" + K3S_KUBECONFIG_MODE: "{{ k8s_config_mode }}" + +k8s_install_bootstrap: >- + server + +k8s_install_server: >- + server + +k8s_install_agent: >- + agent diff --git a/vars/types/rke2.yml b/vars/types/rke2.yml new file mode 100644 index 0000000..5d50cb2 --- /dev/null +++ b/vars/types/rke2.yml @@ -0,0 +1,41 @@ +--- +# See https://docs.rke2.io/ + +# define k8s_version to deploy a specific version +# channel: stable, latest, testing +k8s_install_url: https://get.rke2.io +k8s_channel_url: https://update.rke2.io/v1-release/channels +k8s_cmd_path: /usr/bin + +# rke2 server listens on a dedicatged port for new nodes to register +k8s_supervisor_port: 9345 + +# canal, cilium, calico, flannel +k8s_cni_type: canal + +# disable builtin services +# k8s_disable: +# - rke2-coredns +# - rke2-ingress-nginx +# - rke2-metrics-server +# - rke2-snapshot-controller +# - rke2-snapshot-controller-crd +# - rke2-snapshot-validation-webhook + +# kubelet configs +# - "kube-reserved=cpu=500m,memory=1Gi,ephemeral-storage=2Gi" +# - "system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=2Gi" +# - "eviction-hard=memory.available<500Mi,nodefs.available<10%" +k8s_kubelet_args: + - "max-pods={{ k8s_pod_limit }}" + +k8s_env: + INSTALL_RKE2_CHANNEL_URL: "{{ k8s_channel_url }}" + INSTALL_RKE2_CHANNEL: "{{ k8s_channel }}" + + # will attempt to download from channel if not specified + INSTALL_RKE2_VERSION: "{{ k8s_version | d() }}" + + # server or agent + #INSTALL_RKE2_TYPE: "{{ k8s_channel }}" + \ No newline at end of file