organize shared manifests and tasks, clean up default values, add base64 filter for template based secrets, add handling for locally stores secrets, add helm chart support, add manifest support, general fixes and updates.

.gitignore

@@ -0,0 +1 @@
+__pycache__

README.md

@@ -1,16 +1,28 @@
 # ansible-roles-k8s
 
- - https://docs.k3s.io/
- - https://docs.rke2.io/
- - https://kube-vip.io/
- - https://github.com/sbstp/kubie
- - https://kubernetes.io/docs/tasks/tools/
+- https://docs.k3s.io/
+- https://docs.rke2.io/
+- https://kube-vip.io/
+- https://github.com/sbstp/kubie
+- https://kubernetes.io/docs/tasks/tools/
 
 ## Requirements
 
-Install `yq` on the local system, this is required for the kubectl formatting handler which places an updated kubeconfig in the local ~/.kube
+There is an included helper script to install common tools `scripts/get-kube-tools.sh`
+
+  - `yq` required on the local system for the kubectl formatting task which places an updated kubeconfig in the local ~/.kube
+
+  - `kubectl` required on the local system for basic cluster mangement and application of locally stored manifests or secrets
+
+  - `helm` required on the local system for helm deployments that use locally stored value files, otherwise this is handled on the bootstrap node
+
+  - `kubie` recommened on the local system for context management after deployment
+
+
+## Setup
+
+There is a helper script `scripts/token-vault.sh` which pre-generates a cluster token and places it in an encrypted vault file
 
-Recommended `kubie` for context management after deployment
 
 ## Cluster Example
 
@@ -108,7 +120,13 @@ kubectl get pods,svc,ds --all-namespaces
 Deploy
 
 ```
-ansible-playbook -i hosts site.yml --tags=firewalld,k8s --limit=somehost
+ansible-playbook -i hosts site.yml --tags=firewalld,k8s --limit=k8s_somecluster
+```
+
+Adding a node, simply add the new host to the cluster group with its defined role and deploy
+
+```
+ansible-playbook -i hosts site.yml --tags=firewalld,k8s --limit=just_the_new_host
 ```
 
 Remove firewall role

defaults/main.yml

@@ -9,44 +9,73 @@ k8s_type: k3s
 
 k8s_cluster_name: default
 k8s_cluster_url: localhost
+
+# Additionally define k8s_external_ip to provide a specific node an external route
 k8s_node_ip: "{{ ansible_host }}"
 
 # paths
-k8s_install_script: /usr/local/bin/{{ k8s_type }}-install.sh
-k8s_config_path: "/etc/rancher/{{ k8s_type }}"
-k8s_manifests_path: "/var/lib/rancher/{{ k8s_type }}/server/manifests/"
+# used for placing nm related configs
 k8s_nm_path: /etc/NetworkManager/conf.d
+
+# used by k8s binaries, depends on installation method: rpm vs tar
 k8s_cmd_path: /usr/local/bin
 
+# location of install scripts and other tools
+k8s_install_path: /usr/local/bin
+
+k8s_install_script: "{{ k8s_install_path }}/{{ k8s_type }}-install.sh"
+k8s_manifests_path: "/var/lib/rancher/{{ k8s_type }}/server/manifests/"
+k8s_config_path: "/etc/rancher/{{ k8s_type }}"
+
+k8s_helm_install_url: https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+k8s_helm_install_script: "{{ k8s_install_path }}/get_helm.sh"
+
+# apply CriticalAddonsOnly:NoExecute to control plane nodes
+k8s_taint_servers: false
+
+# shared k8s api port
+k8s_api_port: 6443
+
+# rke2 server listens on a dedicatged port for new nodes to register
+k8s_supervisor_port: 9345
+
 # sysctl set fs.inotify.max_user_instances
 k8s_inotify_max: 1024
 
 # hardcoded kublet default value is 110
 k8s_pod_limit: 110
 
-# overriden by vars/sysetms/
-k8s_selinux: false
-
-# if the host is using network manager, overriden by vars/sysetms/ 
-k8s_has_nm: false
-
 # if the host is using an http proxy for external access
 k8s_http_proxy: false
 
 # kubeconfig chmod
 k8s_config_mode: 600
 
-# rke2 server listens on a dedicatged port for new nodes to register
-k8s_supervisor_port: 9345
-
-# shared k8s api port
-k8s_api_port: 6443
-
-# misc options
-k8s_debug: false
-k8s_taint_servers: false
 k8s_disable_kube_proxy: false
-k8s_flannel_wireguard: false
+k8s_debug: false
+
+k8s_kubelet_args:
+ - "max-pods={{ k8s_pod_limit }}"
+
+# cluster issuers
+# k8s_cluster_issuers:
+#  - name: letsencrypt-prod
+#    url: https://acme-v02.api.letsencrypt.org/directory
+#    solvers: 
+#      - type: http
+#        ingress: nginx
+#      - type: dns
+#        provider: cloudflare
+#        tokenref: apiTokenSecretRef
+#        secret_name: cloudflare-api-token
+#        secret_ley: api-token
+
+# cluster secrets
+# k8s_secrets:
+#  - name: cloudflare-api-token
+#    namespace: cert-manager
+#    data: api-token
+#    value: ZG9wX3Y...
 
 # k8s_kubelet_args
 #  - "kube-reserved=cpu=500m,memory=1Gi,ephemeral-storage=2Gi"
@@ -54,12 +83,14 @@ k8s_flannel_wireguard: false
 #  - "eviction-hard=memory.available<500Mi,nodefs.available<10%"
 #  - "max-pods={{ k8s_pod_limit }}"
 #  - "v=2"
-k8s_kubelet_args:
- - "max-pods={{ k8s_pod_limit }}"
-
 
 # Define
 
+# Default is assumed false, set by vars/sysetms/
+# k8s_selinux: false
+
+# k8s_acme_email
+
 # you can pre-generate this ina vault with the token.sh script
 # k8s_cluster_token
 
@@ -102,12 +133,18 @@ k8s_kubelet_args:
 
 # RKE2
 
+# Default is false, if the host is using network manager, overriden by vars/sysetms/ 
+# k8s_has_nm: true
+
 # canal, cilium, calico, flannel
 # k8s_cni_type: canal
 
 # apply manifest overrides
 # k8s_cni_manifest_overrides: true
 
+# when using canal enable wg backend
+# k8s_canal_wireguard: true
+
 # cilium
 # k8s_cilium_hubble: true
 # k8s_cilium_eni: true

files/get-kube-tools.sh

@@ -1,17 +0,0 @@
-#!/bin/sh
-
-INSTALL_PATH="/usr/local/bin/"
-INSTALL_ARCH="amd64"
-KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt)
-KUBIE_VERSION="latest"
-YQ_VERSION="latest"
-
-wget -qO ${INSTALL_PATH}/kubectl https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${INSTALL_ARCH}/kubectl
-chmod a+x ${INSTALL_PATH}/kubectl
-
-wget -qO ${INSTALL_PATH}/kubie https://github.com/sbstp/kubie/releases/${KUBIE_VERSION}/download/kubie-linux-${INSTALL_ARCH} 
-chmod a+x ${INSTALL_PATH}/kubie
-
-wget -qO ${INSTALL_PATH}/yq https://github.com/mikefarah/yq/releases/${YQ_VERSION}/download/yq_linux_${INSTALL_ARCH}
-chmod a+x ${INSTALL_PATH}/yq
-

filter_plugins/base64_filters.py

@@ -0,0 +1,10 @@
+import base64
+
+def base64_encode(string):
+    return base64.b64encode(string.encode('utf-8')).decode('utf-8')
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            'base64_encode': base64_encode
+        }

handlers/main.yml

@@ -1,11 +0,0 @@
----
-
-- name: Update k8s Local Config
-  listen: "k8s-update-local-config"
-  delegate_to: localhost
-  become: false
-  ansible.builtin.shell: |
-    yq e '.clusters[].name = "{{ k8s_cluster_name }}"' -i ~/.kube/config-{{ k8s_cluster_name }}.yaml
-    yq e '.contexts[].name = "{{ k8s_cluster_context | d(k8s_cluster_name) }}"' -i ~/.kube/config-{{ k8s_cluster_name }}.yaml
-    yq e '(.clusters[] | select(.name == "{{ k8s_cluster_name }}")).cluster.server = "https://{{ k8s_cluster_url }}:{{ k8s_api_port }}"' -i ~/.kube/config-{{ k8s_cluster_name }}.yaml
-    yq e '(.contexts[] | select(.name == "{{ k8s_cluster_name }}")).context.cluster = "{{ k8s_cluster_name }}"' -i ~/.kube/config-{{ k8s_cluster_name }}.yaml

scripts/get-kube-tools.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+INSTALL_PATH="/usr/local/bin"
+INSTALL_ARCH="amd64"
+KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt)
+KUBIE_VERSION="latest"
+YQ_VERSION="latest"
+HELM_URL="https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3"
+
+sudo wget -qO ${INSTALL_PATH}/kubectl https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${INSTALL_ARCH}/kubectl
+sudo chmod a+x ${INSTALL_PATH}/kubectl
+
+sudo wget -qO ${INSTALL_PATH}/kubie https://github.com/sbstp/kubie/releases/${KUBIE_VERSION}/download/kubie-linux-${INSTALL_ARCH} 
+sudo chmod a+x ${INSTALL_PATH}/kubie
+
+sudo wget -qO ${INSTALL_PATH}/yq https://github.com/mikefarah/yq/releases/${YQ_VERSION}/download/yq_linux_${INSTALL_ARCH}
+sudo chmod a+x ${INSTALL_PATH}/yq
+
+curl -fsSL -o get_helm.sh ${HELM_URL}
+chmod 700 get_helm.sh
+./get_helm.sh

scripts/get-secret.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# env expected to be supplied via ansible task
+# PLAYBOOK_PATH
+# KUBECONTEXT
+# SECRET
+
+KUBECONF="$HOME/.kube/config-${KUBECONTEXT}.yaml"
+SECRET_FILE="${PLAYBOOK_DIR}/files/manifests/${SECRET}"
+
+apply_secret() {
+  kubectl apply --kubeconfig="${KUBECONF}" --context="${KUBECONTEXT}" -f "$1"
+}
+
+if ansible-vault view "${SECRET_FILE}" &> /dev/null; then
+  ansible-vault decrypt --output=- "${SECRET_FILE}" | apply_secret -
+else
+  apply_secret "${SECRET_FILE}"
+fi

tasks/k3s/config.yml

@@ -3,6 +3,6 @@
   # PRE-DEPLOY
   # - name: template k3s kubelet config
   #  ansible.builtin.template:
-  #    src: "templates/k3s-kubelet.config.j2"
+  #    src: "templates/k3s/kubelet.config.j2"
   #    dest: "/etc/rancher/k3s/kubelet.config"
   #    mode: 0644

tasks/main.yml

@@ -32,8 +32,9 @@
 #
 # CREATE CLUSTER
 #
-- name: Cluster Creation
-  tags: k8s
+- name: Create Cluster
+  tags: 
+    - k8s
   block:
   
   - name: load server node taints
@@ -71,7 +72,7 @@
   
   - name: template cluster config
     ansible.builtin.template:
-      src: "templates/{{ k8s_type }}-config.yaml.j2"
+      src: "templates/{{ k8s_type }}/config.yaml.j2"
       dest: "{{ k8s_config_path }}/config.yaml"
       mode: 0600
     tags:
@@ -83,39 +84,54 @@
       - k8s-config
 
   # DEPLOY CLUSTER
-  - name: beging cluster creation
+  - name: begining cluster creation
     ansible.builtin.include_tasks: "{{ k8s_type }}/main.yml"
 
-  # KUBE CONFIG
-  - name: fetch kube config
-    ansible.builtin.fetch:
-      src: "{{ k8s_config_path }}/{{ k8s_type }}.yaml"
-      dest: "~/.kube/config-{{ k8s_cluster_name }}.yaml"
-      flat: yes
-    notify: 
-      - k8s-update-local-config
-    when: 
+  # END Cluster Creation
+  when: 
+    - k8s_action == "create"
+
+#
+# POST-DEPLOY
+#
+- name: Post Deployments
+  tags: 
+    - k8s
+    - k8s-post-deploy
+  block:
+
+  - name: include kubeconf block
+    ansible.builtin.include_tasks: "shared/kubeconf.yml"
+    when:
       - k8s_node_type == "bootstrap"
     tags:
       - k8s-get-kubeconf
-  
-  # DEPLOY MANIFESTS
-  - name: apply manifests
-    ansible.builtin.get_url:
-      url: "{{ item.url }}"
-      timeout: 120
-      dest: "{{ k8s_manifests_path }}"
-      owner: root
-      group: root
-      mode: 0644
-    loop: "{{ k8s_manifests }}"
+
+  - name: include secret block
+    ansible.builtin.include_tasks: "shared/secrets.yml"
+    when:
+      - k8s_node_type == "bootstrap"
+      - k8s_secrets is defined
+    tags:
+      - k8s-apply-secrets
+
+  - name: include manifest block
+    ansible.builtin.include_tasks: "shared/manifests.yml"
     when:
       - k8s_node_type == "bootstrap"
       - k8s_manifests is defined
     tags:
       - k8s-apply-manifests
 
-  # END Cluster Creation
+  - name: include chart block
+    ansible.builtin.include_tasks: "shared/charts.yml"
+    when:
+      - k8s_node_type == "bootstrap"
+      - k8s_charts is defined
+    tags:
+      - k8s-apply-charts
+
+  # END Post Deployments
   when: 
     - k8s_action == "create"
 
@@ -123,7 +139,7 @@
 # DESTORY CLUSTER
 #
 # this is very dangerous and should be handled with care when not actively testing with disposable cluster iterations
-- name: Destroy K8s cluster
+- name: Destroy Cluster
   tags: k8s
   block:
 

tasks/rke2/config.yml

@@ -9,11 +9,11 @@
   # CANAL NM CONFIG
   - name: template nm canal config
     ansible.builtin.template:
-      src: "templates/{{ k8s_type }}-canal.conf.j2"
+      src: "templates/{{ k8s_type }}/canal.conf.j2"
       dest: "{{ k8s_nm_path }}/{{ k8s_type }}-canal.conf"
       mode: 0600
     when:
       - k8s_cni_type is not defined or k8s_cni_type == "canal"
-      - k8s_has_nm
+      - k8s_has_nm is defined and k8s_has_nm
     tags:
       - k8s-config

tasks/rke2/main.yml

@@ -9,7 +9,7 @@
 
   - name: rke2 template cni manifest override
     ansible.builtin.template:
-      src: "templates/{{ k8s_type }}-{{ k8s_cni_type | d('canal') }}-config.yaml.j2"
+      src: "templates/{{ k8s_type }}/{{ k8s_cni_type | d('canal') }}-config.yaml.j2"
       dest: "{{ k8s_manifests_path }}/{{ k8s_type }}-{{ k8s_cni_type | d('canal') }}-config.yaml"
       mode: 0600
     when:
@@ -17,7 +17,7 @@
       - k8s_node_type == "bootstrap"
 
   - name: rke2 start bootstrap node
-    ansible.builtin.include_tasks: start.yml
+    ansible.builtin.include_tasks: shared/start.yml
     when:
       - k8s_node_type == "bootstrap"
 
@@ -37,6 +37,6 @@
 
   # POST-DEPLOY
   - name: rke2 start additional nodes
-    ansible.builtin.include_tasks: start.yml
+    ansible.builtin.include_tasks: shared/start.yml
     when:
       - k8s_node_type != "bootstrap"

tasks/rke2/proxy.yml

@@ -42,8 +42,8 @@
 
   - name: template rke2 http proxy
     ansible.builtin.template:
-      src: "templates/{{ k8s_type }}-proxy.j2"
-      dest: "/etc/default/rke2-{{ node_type }}"
+      src: "templates/{{ k8s_type }}/proxy.j2"
+      dest: "/etc/default/{{ k8s_type }}-{{ node_type }}"
       mode: 0644
     when:
       - http_proxy.stdout != ""

tasks/shared/charts.yml

@@ -0,0 +1,37 @@
+---
+- name: begining chart deployments
+  run_once: true
+  tags: 
+    - k8s
+    - k8s-apply-charts
+  block:
+
+  - name: download helm install script
+    ansible.builtin.get_url:
+      url: "{{ k8s_helm_install_url }}"
+      timeout: 120
+      dest: "{{ k8s_helm_install_script }}"
+      owner: root
+      group: root
+      mode: 0700
+  
+  - name: install helm
+    ansible.builtin.shell: "{{ k8s_helm_install_script }}"
+
+  - name: Add chart repos
+    kubernetes.core.helm_repository:
+      name: "{{ item.repo_name }}"
+      repo_url: "{{ item.repo_url }}"
+    loop: "{{ k8s_charts }}"
+    when:
+      - item.repo_name is defined
+      - item.repo_url is defined
+  
+  - name: apply helm charts
+    ansible.builtin.shell: |
+      helm repo update
+      helm upgrade --kubeconfig {{ k8s_config_path }}/{{ k8s_type }}.yaml --namespace {{ item.namespace | d('default') }} --create-namespace --install {{ item.name }} {{ item.chart }} {% if item.chart_version is defined %}--version {{ item.chart_version }}{% endif %} {% if item.settings is defined %}{% for setting in item.settings %}--set {{ setting.key }}={{ setting.value }} {% endfor %}{% endif %}  
+    loop: "{{ k8s_charts }}"
+    when:
+      - item.name is defined
+      - item.chart is defined

tasks/shared/kubeconf.yml

@@ -0,0 +1,23 @@
+---
+- name: fetch and update kubeconf
+  run_once: true
+  tags: 
+    - k8s
+    - k8s-get-kubeconf
+  block:
+
+  - name: fetch kubeconfig
+    ansible.builtin.fetch:
+      src: "{{ k8s_config_path }}/{{ k8s_type }}.yaml"
+      dest: "~/.kube/config-{{ k8s_cluster_name }}.yaml"
+      flat: yes
+
+  - name: Update k8s Local Config
+    delegate_to: localhost
+    connection: local
+    become: false
+    ansible.builtin.shell: |
+      yq e '.clusters[].name = "{{ k8s_cluster_name }}"' -i ~/.kube/config-{{ k8s_cluster_name }}.yaml
+      yq e '.contexts[].name = "{{ k8s_cluster_context | d(k8s_cluster_name) }}"' -i ~/.kube/config-{{ k8s_cluster_name }}.yaml
+      yq e '(.clusters[] | select(.name == "{{ k8s_cluster_name }}")).cluster.server = "https://{{ k8s_cluster_url }}:{{ k8s_api_port }}"' -i ~/.kube/config-{{ k8s_cluster_name }}.yaml
+      yq e '(.contexts[] | select(.name == "{{ k8s_cluster_name }}")).context.cluster = "{{ k8s_cluster_name }}"' -i ~/.kube/config-{{ k8s_cluster_name }}.yaml

tasks/shared/manifests.yml

@@ -0,0 +1,42 @@
+---
+- name: begining chart deployments
+  run_once: true
+  tags: 
+    - k8s
+    - k8s-apply-manifests
+  block:
+
+  - name: apply remote manifests
+    ansible.builtin.get_url:
+      url: "{{ item.source }}"
+      timeout: 120
+      dest: "{{ k8s_manifests_path }}/{{ item.name }}.yaml"
+      owner: root
+      group: root
+      mode: 0600
+    loop: "{{ k8s_manifests }}"
+    when:
+      - item.type == "url"
+      - item.source is defined
+
+  - name: apply local manifests
+    ansible.builtin.copy:
+      src: "manifests/{{ item.source }}"
+      dest: "{{ k8s_manifests_path }}/{{ item.name }}.yaml"
+      owner: root
+      group: root
+      mode: 0600
+    loop: "{{ k8s_manifests }}"
+    when:
+      - item.type == "file"
+      - item.source is defined
+
+  - name: apply template manifests
+    ansible.builtin.template:
+      src: "templates/{{ item.source }}.j2"
+      dest: "{{ k8s_manifests_path }}/{{ item.name }}.yaml"
+      mode: 0600
+    loop: "{{ k8s_manifests }}"
+    when:
+      - item.type == "template"
+      - item.source is defined

tasks/shared/secrets.yml

@@ -0,0 +1,34 @@
+---
+- name: begining chart deployments
+  run_once: true
+  no_log: true
+  tags: 
+    - k8s
+    - k8s-apply-secrets
+  block:
+
+  - name: apply template based secrets
+    ansible.builtin.template:
+      src: "templates/shared/secret.yaml.j2"
+      dest: "{{ k8s_manifests_path }}/{{ item.name }}-secret.yaml"
+      mode: 0600
+    loop: "{{ k8s_secrets }}"
+    when:
+      - item.values is defined
+      - item.type == "template"
+
+  - name: apply locally stored secrets
+    delegate_to: localhost
+    connection: local
+    become: false
+    ansible.builtin.shell: "{{ role_path }}/scripts/get-secret.sh"
+    args:
+      chdir: "{{ playbook_dir }}"
+    environment:
+      PLAYBOOK_DIR: "{{ playbook_dir }}"
+      KUBECONTEXT: "{{ k8s_cluster_name }}"
+      SECRET: "{{ item.source }}"
+    loop: "{{ k8s_secrets }}"
+    when:
+      - item.values is defined
+      - item.type == "file"

tasks/shared/start.yml

@@ -1,6 +1,4 @@
 ---
-
-# handlers dont execute in time so we include this as a task
 - name: enable "{{ k8s_type }}" service
   ansible.builtin.systemd:
     name: "{{ k8s_type }}-{{ node_type }}"

templates/k3s/config.yaml.j2

@@ -18,7 +18,7 @@ tls-san:
 {% elif k8s_cluster_url is defined and k8s_node_type != "agent" -%}
 tls-san: {{ k8s_cluster_url }}
 {% endif %}
-{% if k8s_selinux -%}
+{% if k8s_selinux is defined and k8s_selinux -%}
 selinux: true
 {% endif -%}
 

templates/rke2/canal-config.yaml.j2

@@ -10,7 +10,7 @@ metadata:
 spec:
   valuesContent: |-
     flannel:
-{% if k8s_flannel_wireguard %}
+{% if k8s_canal_wireguard is defined and k8s_canal_wireguard %}
       backend: "wireguard"
 {% else %}
 {% if k8s_cni_interface is defined %}

templates/rke2/config.yaml.j2

@@ -15,7 +15,7 @@ tls-san:
 {% elif k8s_cluster_url is defined and k8s_node_type != "agent" -%}
 tls-san: {{ k8s_cluster_url }}
 {% endif %}
-{% if k8s_selinux -%}
+{% if k8s_selinux is defined and k8s_selinux -%}
 selinux: true
 {% endif -%}
 

templates/shared/clusterissuer-acme.yaml.j2

@@ -0,0 +1,24 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: {{ item.name }}
+spec:
+ acme:
+  server: {{ item.server | d('https://acme-v02.api.letsencrypt.org/directory') }}
+  email: {{ item.email | d(k8s_acme_email) }}
+  privateKeySecretRef:
+    name: {{ item.name }}-prviate-key
+  solvers:
+{% for solver in item.solvers %}
+{% if solver.type == "http" %}
+    - http01:
+        ingress:
+          class: {{ solver.ingress }}
+{% elif solver.type == "dns" %}
+    - dns01:
+        {{ solver.provider }}:
+          {{ solver.tokenref }}:
+            name: {{ solver.secret_name }}
+            key: {{ solver.secret_key }}
+{% endif -%}
+{% endfor -%}

templates/shared/secret.yaml.j2

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ item.name }}
+  namespace: {{ item.namespace | d('default') }}
+data:
+{% for secret in item.secrets %}
+  {{ secret.key }}: {{ secret.value | base64_encode }}
+{% endfor -%}