initial commit

README.md

@@ -0,0 +1,33 @@
+# Ansible Playbook Example
+
+Setup roles
+
+```
+ansible-galaxy install -f -r galaxy-roles/requirements.yml
+```
+
+Create a cluster, this assumes you have real machines in the inventory.
+
+```
+ansible-playbook -i hosts deploy.yml --tags=k8s --limit=k8s_cluster
+```
+
+
+Deploy a specific role, you must have a local context activated, all tasks run locally using existing credentials.
+
+```
+ansible-playbook -i hosts deploy.yml --tags=redis
+```
+
+Select aspects of the k8s roles can be targed via tag
+
+```
+ansible-playbook -i hosts deploy.yml --tags=redis-env
+ansible-playbook -i hosts deploy.yml --tags=redis-deploy
+```
+
+Deployments can be removed
+
+```
+ansible-playbook -i hosts deploy.yml --tags=redis --extra-vars "redis_state=absent" 
+```

deploy.yml

@@ -0,0 +1,31 @@
+---
+- name: Setup environment
+  hosts: all
+  tasks:
+    - name: Load Default values
+      ansible.builtin.include_vars:
+        file: Default.yml
+    - name: Load OS specific values
+      ansible.builtin.include_vars: "{{ item }}"
+      with_first_found:
+        - files:
+            - "{{ ansible_os_family }}-{{ ansible_distribution_major_version }}.yml"
+            - "{{ ansible_os_family }}.yml"
+            - "{{ ansible_distribution }}.yml"
+            - "{{ ansible_system }}.yml"
+          skip: true
+  tags:
+    - always
+
+- name: Setup shared roles
+  hosts: all
+  roles:
+    - role: firewalld
+
+- name: Setup shared services
+  hosts: localhost
+  run_once: true
+  become: false
+  roles:
+    - role: redis-k8s
+    - role: elasticsearch-k8s

files/k8s/k8s-cluster/manifests/fluent-bit.yaml

@@ -0,0 +1,186 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: fluent-bit
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: fluent-bit-auth
+  namespace: fluent-bit
+type: Opaque
+data:
+  username: dXNlcg==
+  password: eHh4LXh4eC14eHgteHh4LXh4eA==
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: fluent-bit
+  namespace: fluent-bit
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: fluent-bit-read
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: fluent-bit-read
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: fluent-bit-read
+subjects:
+  - kind: ServiceAccount
+    name: fluent-bit
+    namespace: fluent-bit
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fluent-bit-config
+  namespace: fluent-bit
+data:
+  fluent-bit.conf: |
+    [SERVICE]
+        Flush        1
+        Log_Level    error
+        Parsers_File parsers.conf
+
+    [INPUT]
+        Name              tail
+        Path              /var/log/containers/*.log
+        Parser            docker
+        Tag               kube.*
+        Mem_Buf_Limit     5MB
+        Skip_Long_Lines   On
+
+    [INPUT]
+        Name              systemd
+        Tag               host.journald
+        Path              /run/log/journal
+        DB                /var/log/flb_journald.db
+        Read_From_Tail    On
+
+    [FILTER]
+        Name              kubernetes
+        Match             kube.*
+        Kube_URL          https://kubernetes.default.svc:443
+        Merge_Log         On
+        K8S-Logging.Exclude Off
+        Labels            On
+        Annotations       On
+
+    [FILTER]
+        Name              grep
+        Match             kube.*
+        Exclude           container_name ingress-nginx
+
+    [FILTER]
+        Name              modify
+        Match             *
+        Add               node ${NODE_NAME}
+
+    [FILTER]
+        Name              modify
+        Match             host.journald
+        Add               log_type host
+        Add               environment production
+
+    [OUTPUT]
+        Name              http
+        Match             *
+        Host              logs.clearwater.20c.dev
+        Port              443
+        URI               /insert/jsonline?_stream_fields=stream,node&_msg_field=log&_time_field=date
+        Format            json_lines
+        json_date_format  iso8601
+        HTTP_User         ${FLUENTBIT_USER}
+        HTTP_Passwd       ${FLUENTBIT_PASSWORD}
+        tls               On
+
+  parsers.conf: |
+    [PARSER]
+        Name              docker
+        Format            json
+        Time_Key          time
+        Time_Format       %Y-%m-%dT%H:%M:%S.%L
+        Time_Keep         On
+
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fluent-bit
+  namespace: fluent-bit
+  labels:
+    k8s-app: fluent-bit
+spec:
+  selector:
+    matchLabels:
+      k8s-app: fluent-bit
+  template:
+    metadata:
+      labels:
+        k8s-app: fluent-bit
+    spec:
+      serviceAccountName: fluent-bit
+      tolerations:
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+          effect: "NoExecute"
+        - key: "node-role.kubernetes.io/control-plane"
+          operator: "Exists"
+          effect: "NoSchedule"
+      containers:
+        - name: fluent-bit
+          image: fluent/fluent-bit:latest
+          volumeMounts:
+            - name: varlog
+              mountPath: /var/log
+            - name: journal
+              mountPath: /run/log/journal
+            - name: config
+              mountPath: /fluent-bit/etc/
+          env:
+            - name: FLUENTBIT_USER
+              valueFrom:
+                secretKeyRef:
+                  name: fluent-bit-auth
+                  key: username
+            - name: FLUENTBIT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: fluent-bit-auth
+                  key: password
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+      hostAliases:
+        - ip: "10.31.101"
+          hostnames:
+            - "logs.somehost"
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: varlog
+          hostPath:
+            path: /var/log
+        - name: journal
+          hostPath:
+            path: /run/log/journal
+        - name: config
+          configMap:
+            name: fluent-bit-config

files/k8s/k8s-cluster/manifests/metallb-config.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: public
+  namespace: metallb-system
+spec:
+  addresses:
+  - x.x.x.x/32
+
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: external
+  namespace: metallb-system
+spec:
+  ipAddressPools:
+  - public

files/manifests/external-dns.yaml

@@ -0,0 +1,67 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: external-dns
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: external-dns
+rules:
+- apiGroups: [""]
+  resources: ["services","endpoints","pods"]
+  verbs: ["get","watch","list"]
+- apiGroups: ["extensions","networking.k8s.io"]
+  resources: ["ingresses"] 
+  verbs: ["get","watch","list"]
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["list"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: external-dns-viewer
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: external-dns
+subjects:
+- kind: ServiceAccount
+  name: external-dns
+  namespace: default
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: external-dns
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: external-dns
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: external-dns
+    spec:
+      serviceAccountName: external-dns
+      containers:
+      - name: external-dns
+        image: registry.k8s.io/external-dns/external-dns:v0.14.2
+        args:
+        - --source=ingress
+        - --domain-filter=apps.vaasl.io
+        - --provider=digitalocean
+        env:
+        - name: DO_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: digitalocean-dns
+              key: access-token

files/manifests/secrets.yaml

@@ -0,0 +1,15 @@
+# This should be encrypted as a vault
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cert-manager
+
+---
+apiVersion: v1
+data:
+  access-token: xxx
+kind: Secret
+metadata:
+  name: digitalocean-dns
+  namespace: cert-manager

files/manifests/test-deployment.yaml

@@ -0,0 +1,63 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  labels:
+    app: nginx
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:latest
+        ports:
+        - containerPort: 80
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-service
+  labels:
+    app: nginx
+spec:
+  type: ClusterIP
+  ports:
+    - port: 80
+      targetPort: 80
+  selector:
+    app: nginx
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: nginx-ingress
+  namespace: default
+  annotations:
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+spec:
+  ingressClassName: nginx
+  tls:
+  - hosts:
+    - x.ebenezer.20c.dev
+    secretName: nginx-tls
+  rules:
+  - host: x.ebenezer.20c.dev
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: nginx-service
+            port:
+              number: 80

files/manifests/wildcard-vaasl-apps.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: wildcard-vaasl-apps
+  namespace: default
+spec:
+  secretName: wildcard-vaasl-apps
+  issuerRef:
+    name: letsencrypt-prod-wild
+    kind: ClusterIssuer
+    group: cert-manager.io
+  commonName: "*.apps.vaasl.io"
+  dnsNames:
+    - "apps.vaasl.io"
+    - "*.apps.vaasl.io"

galaxy-roles/requirements.yml

@@ -0,0 +1,20 @@
+---
+- name: firewalld
+  scm: git
+  src: git@github.com:srwadleigh/ansible-role-firewalld.git
+  version: main
+
+- name: k8s
+  scm: git
+  src: git@github.com:srwadleigh/ansible-role-k8s.git
+  version: main
+
+- name: redis-k8s
+  scm: git
+  src: git@git.vdb.to:ops/redis-k8s.git
+  version: main
+
+- name: elasticsearch-k8s
+  scm: git
+  src: git@git.vdb.to:ops/elasticsearch-k8s.git
+  version: main

group_vars/all/k8s_cluster/firewalld.yaml

@@ -0,0 +1,23 @@
+---
+firewalld_add:
+  - name: internal
+    services:
+      - ssh
+      - http
+      - https
+      - redis
+      - elasticsearch
+    ports:
+      - 6443/tcp
+      - 9100/tcp
+
+  - name: public
+    services:
+      - http
+      - https
+
+firewalld_remove:
+  - name: public
+    services:
+      - dhcpv6-client
+      - ssh

group_vars/all/k8s_cluster/main.yaml

@@ -0,0 +1,94 @@
+---
+# K8S CLUSTER
+k8s_type: rke2
+k8s_cluster_name: k8s-cluster
+k8s_cluster_url: control1.somehost
+
+k8s_taint_servers: true
+k8s_label_agents: true
+
+k8s_cni_manifest_override: true
+k8s_cni_custom_template: canal-config.yaml
+k8s_cni_interface: eth0
+
+k8s_certmanager_version: v1.18.2
+k8s_metallb_version: v0.15.2
+
+k8s_manifests:
+  - name: local-path-storage
+    type: template
+
+  # RKE2 Ingress as a LoadBalancer
+  - name: rke2-ingress-loadbalancer
+    type: file
+
+  # VIP LoadBalancer
+  - name: metallb
+    type: url
+    source: "{{ k8s_metallb_manifest_url }}"
+
+  # TLS
+  - name: cert-manager
+    type: url
+    source: "{{ k8s_certmanager_manifest_url }}"
+
+  - name: letsencrypt-prod
+    type: template
+    source: "{{ k8s_clusterissuer_template }}"
+    server: "{{ k8s_clusterissuer_le_prod_url }}"
+    solvers: 
+      - type: http
+        ingress: nginx
+
+# example using cert-manager helm chart to insert custom values
+# k8s_charts:
+#  - name: "{{ k8s_certmanager_name }}"
+#    namespace: "{{ k8s_certmanager_name }}"
+#    chart_ref: "{{ k8s_certmanager_chart }}"
+#    chart_version: v1.18.2
+#    chart_values:
+#      crds:
+#        enabled: true
+#      config:
+#        featureGates:
+#          # Disable the use of Exact PathType in Ingress resources, to work around a bug in ingress-nginx
+#          # https://github.com/kubernetes/ingress-nginx/issues/11176
+#          ACMEHTTP01IngressPathTypeExact: false
+#      extraEnv:
+#          - { name: HTTP_PROXY,  value: "{{ k8s_http_proxy }}" }
+#          - { name: HTTPS_PROXY, value: "{{ k8s_https_proxy }}" }
+#          - { name: NO_PROXY,    value: "{{ k8s_no_proxy }}" }
+#      cainjector:
+#        extraEnv:
+#          - { name: HTTP_PROXY,  value: "{{ k8s_http_proxy }}" }
+#          - { name: HTTPS_PROXY, value: "{{ k8s_https_proxy }}" }
+#          - { name: NO_PROXY,    value: "{{ k8s_no_proxy }}" }
+#      webhook:
+#        extraEnv:
+#          - { name: HTTP_PROXY,  value: "{{ k8s_http_proxy }}" }
+#          - { name: HTTPS_PROXY, value: "{{ k8s_https_proxy }}" }
+#          - { name: NO_PROXY,    value: "{{ k8s_no_proxy }}" }
+
+# REDIS
+redis_namespace: default
+redis_image: docker.io/redis
+redis_image_tag: latest
+
+redis_sentinel_master: peeringdb-redis
+redis_replicas: 3
+redis_min_available: 2
+
+# ELASTICSEARCH
+elasticsearch_namespace: default
+elasticsearch_image: docker.elastic.co/elasticsearch/elasticsearch
+elasticsearch_image_tag: 9.1.4
+
+elasticsearch_cluster_name: es-cluster
+elasticsearch_replicas: 3
+elasticsearch_min_available: 2
+
+# requiredDuringSchedulingIgnoredDuringExecution,preferredDuringSchedulingIgnoredDuringExecution
+elasticsearch_anti_affinity_type: preferredDuringSchedulingIgnoredDuringExecution
+
+# DoNotSchedule, ScheduleAnyway
+elasticsearch_topology_constraint: ScheduleAnyway

group_vars/all/k8s_cluster/vault.yaml

@@ -0,0 +1,25 @@
+# This should be encrypted as a vault
+---
+k8s_cluster_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+
+elasticsearch_secrets:
+  ELASTIC_PASSWORD: xxx-xxx-xxx-xxx-xxx-xxx
+
+redis_secrets:
+  REDIS_PASSWORD: xxx-xxx-xxx-xxx-xxx-xxx
+
+# cluster secrects could be defined here also
+# k8s_secrets:
+#  - name: digitalocean-dns
+#    type: template
+#    namespace: cert-manager
+#    secrets:
+#      - key: access-token
+#        value: XXX
+#      - key: auth
+#        value: XXX
+#
+#  - name: somes-secret
+#    type: file
+#    source: some-secret.yaml
+

hosts

@@ -0,0 +1,8 @@
+[k8s_cluster]
+control1 k8s_node_type=bootstrap
+agent1 k8s_node_type=agent k8s_pod_limit=1024
+agent2 k8s_node_type=agent k8s_pod_limit=1024
+agent3 k8s_node_type=agent k8s_pod_limit=1024
+
+[k8s:children]
+k8s_cluster