clean up cluster type env, update k3s config template

defaults/main.yml

@@ -6,62 +6,77 @@ k8s_action: create
 
 # k3s | rke2
 k8s_type: k3s
-k8s_channel: stable
 
 k8s_cluster_name: default
 k8s_cluster_url: localhost
-
-# bootstrap | server | agent
-k8s_node_type: bootstrap
 k8s_node_ip: "{{ ansible_host }}"
 
+# paths
+k8s_install_script: /usr/local/bin/{{ k8s_type }}-install.sh
+k8s_config_path: "/etc/rancher/{{ k8s_type }}"
+k8s_cmd_path: /usr/local/bin
+k8s_nm_path: /etc/NetworkManager/conf.d
+k8s_manifests_path: "/var/lib/rancher/{{ k8s_type }}/server/manifests/"
+
 # sysctl set fs.inotify.max_user_instances
 k8s_inotify_max: 1024
 
 # hardcoded kublet default value is 110
 k8s_pod_limit: 110
 
-# we can set this by platform later
+# overriden by vars/sysetms/
 k8s_selinux: false
 
-# if the host is using network manager, see vars/sys/ for overrides
+# if the host is using network manager, overriden by vars/sysetms/ 
 k8s_has_nm: false
 
-# if the host is using an http proxy
+# if the host is using an http proxy for external access
 k8s_http_proxy: false
 
-# cni
+# kubeconfig chmod
-# k8s_cni_type:
-
 k8s_config_mode: 600
+
+# rke2 server listens on a dedicatged port for new nodes to register
+k8s_supervisor_port: 9345
+
+# shared k8s api port
 k8s_api_port: 6443
+
+# misc options
 k8s_debug: false
-k8s_skip_start: false
 k8s_taint_servers: false
 k8s_flannel_wireguard: false
 k8s_disable_kube_proxy: false
+k8s_disable_network_policy: false
 
-# paths
+# k8s_kubelet_args
-k8s_install_script: /usr/local/bin/{{ k8s_type }}-install.sh
+#  - "kube-reserved=cpu=500m,memory=1Gi,ephemeral-storage=2Gi"
-k8s_config_path: "/etc/rancher/{{ k8s_type }}"
+#  - "system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=2Gi"
-k8s_cmd_path: /usr/local/bin
+#  - "eviction-hard=memory.available<500Mi,nodefs.available<10%"
-k8s_nm_path: /etc/NetworkManager/conf.d
+#  - "max-pods={{ k8s_pod_limit }}"
+#  - "v=2"
+k8s_kubelet_args:
+ - "max-pods={{ k8s_pod_limit }}"
 
-k8s_manifests_path: "/var/lib/rancher/{{ k8s_type }}/server/manifests/"
+# Define
+
+# you can pre-generate this ina vault with the token.sh script
+# k8s_cluster_token
+
+# stable, latest, testing, ...
+# k8s_channel: stable
+
+# k8s_version to deploy a specific version
+# k8s_version: v1.27.7+k3s2
+
+# bootstrap | server | agent
+# k8s_node_type: bootstrap
 
 # if defined, install manifests
 # k8s_manifests:
 #  - name: cert-manager
 #    path: https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml
 
-# Override
-# k8s_cluster_name
-# k8s_cluster_url
-
-# Define
-# k8s_cluster_token
-# you can pre-generate this ina vault with the token.sh script
-
 # k8s_node_taints
 # --node-taint CriticalAddonsOnly=true:NoExecute
 # k8s_node_taints:
@@ -69,15 +84,30 @@ k8s_manifests_path: "/var/lib/rancher/{{ k8s_type }}/server/manifests/"
 #    value: true
 #    effect: NoExecute
 
-# these are provided simply for the opportunity to override in cases where some ajustment isnt supported by the config templates
-# k8s_install_bootstrap: >-
-#  server --cluster-init --tls-san {{ k8s_cluster_url }} --node-taint CriticalAddonsOnly=true:NoExecute
-#  {% if k8s_disable is defined %}
-#  {% for disable in k8s_disable %}
-#  --disable={{ disable }}
-#  {% endfor %}
-#  {% endif %}
 
-# k8s_install_agent: >-
+# K3S
-#  agent --kubelet-arg=config=/etc/rancher/k3s/kubelet.config --node-ip={{ ansible_host }}
+
-#  {% if k8s_external_ip is defined %}--node-external-ip={{ k8s_external_ip }}{% endif %}
+# flannel-backend: 'vxlan', 'host-gw', 'wireguard-native', 'none'
+# k8s_flannel_backend: vxlan
+# k8s_flannel_ipv6_masq: false
+# k8s_flannel_external_ip: false
+
+# disable builtin services
+# k8s_disable: 
+#  - traefik
+#  - servicelb
+
+
+# RKE2
+
+# canal, cilium, calico, flannel
+# k8s_cni_type: canal
+
+# disable builtin services
+# k8s_disable: 
+#  - rke2-coredns
+#  - rke2-ingress-nginx
+#  - rke2-metrics-server
+#  - rke2-snapshot-controller
+#  - rke2-snapshot-controller-crd
+#  - rke2-snapshot-validation-webhook

meta/main.yml

@@ -1,38 +1,25 @@
 ---
-dependencies: []
-
 galaxy_info:
-  role_name: k8s
+  author: Shane Wadleigh
-  author: srw
+  description: An Ansible role for configuring nginx and letsencrypt certificates
-  description: Ansible role for configuring k3s and rke2 kubernetes clusters
+  company: 20C
-  company: "NMD, LLC"
+  license: Apache
-  license: "license (BSD, MIT)"
+  min_ansible_version: "2"
-  min_ansible_version: "2.10"
   platforms:
-    - name: Fedora
+  - name: EL
-      versions:
+    versions:
-        - all
+      - all
-    - name: Debian
+  - name: Fedora
-      versions:
+    versions:
-        - buster
+      - all
-        - bullseye
+  - name: Ubuntu
-        - bookworm
+    versions:
-    - name: Ubuntu
+      - all
-      versions:
+  - name: Debian
-        - bionic
+    versions:
-        - focal
+      - all
-        - jammy
+dependencies: []
-    - name: Alpine
+#dependencies:
-      version:
+#  - name: common
-        - all
+#    src: https://github.com/your-username/common-role
-    - name: ArchLinux
+#    version: master  # You can specify a specific tag or branch
-      versions:
-        - all
-  galaxy_tags:
-    - server
-    - system
-    - containers
-    - kubernetes
-    - k8s
-    - k3s
-    - rke2

tasks/k3s/config.yml

@@ -1,8 +1,8 @@
 ---
 
   # PRE-DEPLOY
-  - name: template k3s kubelet config
+  # - name: template k3s kubelet config
-    ansible.builtin.template:
+  #  ansible.builtin.template:
-      src: "templates/k3s-kubelet.config.j2"
+  #    src: "templates/k3s-kubelet.config.j2"
-      dest: "/etc/rancher/k3s/kubelet.config"
+  #    dest: "/etc/rancher/k3s/kubelet.config"
-      mode: 0644
+  #    mode: 0644

tasks/k3s/main.yml

@@ -3,20 +3,20 @@
   # BOOTSTRAP
   - name: k3s boostrap initial server node
     ansible.builtin.shell: "{{ k8s_install_script }}"
-    environment: "{{ k8s_env | combine({'INSTALL_K3S_EXEC': '{{ k8s_install_bootstrap }}'}) }}"
+    environment: "{{ k8s_env }}"
     when:
       - k8s_node_type == "bootstrap"
 
   # ADD SERVERS
   - name: k3s add additional server nodes
     ansible.builtin.shell: "{{ k8s_install_script }}"
-    environment: "{{ k8s_env | combine({'INSTALL_K3S_EXEC': '{{ k8s_install_server }}'}) }}"
+    environment: "{{ k8s_env }}"
     when:
       - k8s_node_type == "server"
 
   # ADD AGENTS
   - name: k3s add agent nodes
     ansible.builtin.shell: "{{ k8s_install_script }}"
-    environment: "{{ k8s_env | combine({'INSTALL_K3S_EXEC': '{{ k8s_install_agent }}'}) }}"
+    environment: "{{ k8s_env }}"
     when:
       - k8s_node_type == "agent"

tasks/main.yml

@@ -10,10 +10,10 @@
       local_user: "{{ lookup('env', 'USER') }}"
       delegate_to: localhost
 
-  # useful to set this bc k8s_node_type has a special value bootstrap which is not an actual type
+  # resolve actual node type, boostrap is not recognized 
   - name: set true node type
     set_fact:
-      node_type: "{{ 'agent' if k8s_node_type == 'agent' else 'server' }}"
+      node_type: "{{ 'server' if k8s_node_type == 'bootstrap' else k8s_node_type }}"
 
   - name: load type specific values
     ansible.builtin.include_vars:
@@ -50,7 +50,7 @@
 
   - name: download install script
     ansible.builtin.get_url:
-      url: "{{ k8s_install_url }}"
+      url: "{{ k8s_install_url | d(k8s_default_install_url) }}"
       timeout: 120
       dest: "{{ k8s_install_script }}"
       owner: root

tasks/rke2/config.yml

@@ -13,7 +13,7 @@
       dest: "{{ k8s_nm_path }}/{{ k8s_type }}-canal.conf"
       mode: 0600
     when:
-      - k8s_cni_type == "canal"
+      - k8s_cni_type == "canal" or k8s_cni_type is not defined
       - k8s_has_nm
     tags:
       - k8s-config

tasks/rke2/main.yml

@@ -3,14 +3,14 @@
   # BOOTSTRAP
   - name: rke2 boostrap initial server node
     ansible.builtin.shell: "{{ k8s_install_script }}"
-    environment: "{{ k8s_env | combine({'INSTALL_RKE2_TYPE': 'server'}) }}"
+    environment: "{{ k8s_env }}"
     when:
       - k8s_node_type == "bootstrap"
 
   - name: rke2 template cni manifests
     ansible.builtin.template:
-      src: "templates/{{ k8s_type }}-{{ k8s_cni_type }}-config.yaml.j2"
+      src: "templates/{{ k8s_type }}-{{ k8s_cni_type | d('canal') }}-config.yaml.j2"
-      dest: "{{ k8s_manifests_path }}/{{ k8s_type }}-{{ k8s_cni_type }}-config.yaml"
+      dest: "{{ k8s_manifests_path }}/{{ k8s_type }}-{{ k8s_cni_type | d('canal') }}-config.yaml"
       mode: 0600
     when:
       - k8s_node_type == "bootstrap"
@@ -23,14 +23,14 @@
   # ADD SERVERS
   - name: rke2 add additional server nodes
     ansible.builtin.shell: "{{ k8s_install_script }}"
-    environment: "{{ k8s_env | combine({'INSTALL_RKE2_TYPE': 'server'}) }}"
+    environment: "{{ k8s_env }}"
     when:
       - k8s_node_type == "server"
 
   # ADD AGENTS
   - name: rke2 add agent nodes
     ansible.builtin.shell: "{{ k8s_install_script }}"
-    environment: "{{ k8s_env | combine({'INSTALL_RKE2_TYPE': 'agent'}) }}"
+    environment: "{{ k8s_env }}"
     when:
       - k8s_node_type == "agent"
 

templates/k3s-config.yaml.j2

@@ -22,6 +22,13 @@ tls-san: {{ k8s_cluster_url }}
 selinux: true
 {% endif -%}
 
+{% if k8s_disable_kube_proxy and k8s_node_type != "agent" -%}
+disable-kube-proxy: true
+{% endif -%}
+{% if k8s_disable_network_policy and k8s_node_type != "agent" -%}
+disable-network-policy: true
+{% endif -%}
+
 {% if k8s_disable is defined and k8s_node_type != "agent" %}
 # disable builtin services
 {% for disable in k8s_disable %}
@@ -30,6 +37,13 @@ disable: {{ disable }}
 {% endif -%}
 {% endif %}
 
+{% if k8s_flannel_backend is defined and k8s_node_type != "agent" -%}
+# cofigure or disable flannel cni
+flannel-backend: {{ k8s_flannel_backend | d('vxlan') }}
+flannel-ipv6-masq: {{ k8s_flannel_ipv6_masq | d('false') }}
+flannel-external-ip: {{ k8s_flannel_external_ip | d('false') }}
+{% endif %}
+
 # node network
 {% if k8s_node_ip is defined -%}
 node-ip: {{ k8s_node_ip }}
@@ -38,13 +52,6 @@ node-ip: {{ k8s_node_ip }}
 node-external-ip: {{ k8s_external_ip }}
 {% endif -%}
 
-{% if k8s_flannel_backend is defined and k8s_node_type != "agent" -%}
-# cofigure or disable flannel cni
-flannel-backend: {{ k8s_flannel_backend }}
-flannel-ipv6-masq: {{ k8s_flannel_ipv6_masq }}
-flannel-external-ip: {{ k8s_flannel_external_ip }}
-{% endif %}
-
 {% if k8s_node_taints is defined -%}
 # initial node taints
 {% for taint in k8s_node_taints -%}

vars/types/k3s.yml

@@ -1,45 +1,13 @@
 ---
 # See https://docs.k3s.io/
 
-# define k8s_version to deploy a specific version
+k8s_default_install_url: https://get.k3s.io
-# channel: stable, latest, testing
+k8s_default_channel_url: https://update.k3s.io/v1-release/channels
-k8s_install_url: https://get.k3s.io
-k8s_channel_url: https://update.k3s.io/v1-release/channels
-
-# cluster network (cni)
-# flannel-backend: 'vxlan', 'host-gw', 'wireguard-native', 'none'
-k8s_flannel_backend: vxlan
-k8s_flannel_ipv6_masq: false
-k8s_flannel_external_ip: false
-
-# disable builtin services
-k8s_disable: 
-  - "traefik"
-
-# kubelet configs
-#  - "kube-reserved=cpu=500m,memory=1Gi,ephemeral-storage=2Gi"
-#  - "system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=2Gi"
-#  - "eviction-hard=memory.available<500Mi,nodefs.available<10%"
-k8s_kubelet_args:
- - config=/etc/rancher/k3s/kubelet.config
 
 k8s_env:
-  INSTALL_K3S_CHANNEL_URL: "{{ k8s_channel_url }}"
+  #K3S_KUBECONFIG_MODE: "{{ k8s_config_mode }}"
-  INSTALL_K3S_CHANNEL: "{{ k8s_channel }}"
+  INSTALL_K3S_SKIP_START: "{{ k8s_skip_start | d('false') }}"
-  INSTALL_K3S_SKIP_START: "{{ k8s_skip_start }}"
+  INSTALL_K3S_CHANNEL_URL: "{{ k8s_channel_url | d(k8s_default_channel_url) }}"
-
+  INSTALL_K3S_CHANNEL: "{{ k8s_channel | d('stable') }}"
-  # will attempt to download from channel if not specified
   INSTALL_K3S_VERSION: "{{ k8s_version | d() }}"
-
+  INSTALL_K3S_EXEC: "{{ node_type }} {{ node_install_args | d() }}"
-  # there is some consider for where the token lives after initial node creation, this could get pruned from env or config most likely
-  #K3S_TOKEN: "{{ k8s_cluster_token }}"
-  K3S_KUBECONFIG_MODE: "{{ k8s_config_mode }}"
-
-k8s_install_bootstrap: >-
-  server
-
-k8s_install_server: >-
-  server
-
-k8s_install_agent: >-
-  agent

vars/types/rke2.yml

@@ -1,41 +1,11 @@
 ---
 # See https://docs.rke2.io/
 
-# define k8s_version to deploy a specific version
+k8s_default_install_url: https://get.rke2.io
-# channel: stable, latest, testing
+k8s_default_channel_url: https://update.rke2.io/v1-release/channels
-k8s_install_url: https://get.rke2.io
-k8s_channel_url: https://update.rke2.io/v1-release/channels
-k8s_cmd_path: /usr/bin
-
-# rke2 server listens on a dedicatged port for new nodes to register
-k8s_supervisor_port: 9345
-
-# canal, cilium, calico, flannel
-k8s_cni_type: canal
-
-# disable builtin services
-# k8s_disable: 
-#  - rke2-coredns
-#  - rke2-ingress-nginx
-#  - rke2-metrics-server
-#  - rke2-snapshot-controller
-#  - rke2-snapshot-controller-crd
-#  - rke2-snapshot-validation-webhook
-
-# kubelet configs
-#  - "kube-reserved=cpu=500m,memory=1Gi,ephemeral-storage=2Gi"
-#  - "system-reserved=cpu=500m,memory=1Gi,ephemeral-storage=2Gi"
-#  - "eviction-hard=memory.available<500Mi,nodefs.available<10%"
-k8s_kubelet_args:
- - "max-pods={{ k8s_pod_limit }}"
 
 k8s_env:
-  INSTALL_RKE2_CHANNEL_URL: "{{ k8s_channel_url }}"
+  INSTALL_RKE2_CHANNEL_URL: "{{ k8s_channel_url | d(k8s_default_channel_url) }}"
-  INSTALL_RKE2_CHANNEL: "{{ k8s_channel }}"
+  INSTALL_RKE2_CHANNEL: "{{ k8s_channel | d('stable') }}"
-
-  # will attempt to download from channel if not specified
   INSTALL_RKE2_VERSION: "{{ k8s_version | d() }}"
-
+  INSTALL_RKE2_TYPE: "{{ node_type }} {{ node_install_args | d() }}"
-  # server or agent
-  #INSTALL_RKE2_TYPE: "{{ k8s_channel }}"
-