From a38f3ac1cfef225785d45f82fcdde9d9f6a2fef1 Mon Sep 17 00:00:00 2001 From: srw Date: Wed, 1 Oct 2025 04:44:52 +0000 Subject: [PATCH] initial commit --- README.md | 33 ++++ deploy.yml | 31 +++ .../k8s/k8s-cluster/manifests/fluent-bit.yaml | 186 ++++++++++++++++++ .../k8s-cluster/manifests/metallb-config.yaml | 19 ++ files/manifests/external-dns.yaml | 67 +++++++ files/manifests/secrets.yaml | 15 ++ files/manifests/test-deployment.yaml | 63 ++++++ files/manifests/wildcard-vaasl-apps.yaml | 16 ++ galaxy-roles/requirements.yml | 20 ++ group_vars/all/k8s_cluster/firewalld.yaml | 23 +++ group_vars/all/k8s_cluster/main.yaml | 94 +++++++++ group_vars/all/k8s_cluster/vault.yaml | 25 +++ hosts | 8 + 13 files changed, 600 insertions(+) create mode 100644 README.md create mode 100644 deploy.yml create mode 100644 files/k8s/k8s-cluster/manifests/fluent-bit.yaml create mode 100644 files/k8s/k8s-cluster/manifests/metallb-config.yaml create mode 100644 files/manifests/external-dns.yaml create mode 100644 files/manifests/secrets.yaml create mode 100644 files/manifests/test-deployment.yaml create mode 100644 files/manifests/wildcard-vaasl-apps.yaml create mode 100644 galaxy-roles/requirements.yml create mode 100644 group_vars/all/k8s_cluster/firewalld.yaml create mode 100644 group_vars/all/k8s_cluster/main.yaml create mode 100644 group_vars/all/k8s_cluster/vault.yaml create mode 100644 hosts diff --git a/README.md b/README.md new file mode 100644 index 0000000..af29804 --- /dev/null +++ b/README.md @@ -0,0 +1,33 @@ +# Ansible Playbook Example + +Setup roles + +``` +ansible-galaxy install -f -r galaxy-roles/requirements.yml +``` + +Create a cluster, this assumes you have real machines in the inventory. + +``` +ansible-playbook -i hosts deploy.yml --tags=k8s --limit=k8s_cluster +``` + + +Deploy a specific role, you must have a local context activated, all tasks run locally using existing credentials. + +``` +ansible-playbook -i hosts deploy.yml --tags=redis +``` + +Select aspects of the k8s roles can be targed via tag + +``` +ansible-playbook -i hosts deploy.yml --tags=redis-env +ansible-playbook -i hosts deploy.yml --tags=redis-deploy +``` + +Deployments can be removed + +``` +ansible-playbook -i hosts deploy.yml --tags=redis --extra-vars "redis_state=absent" +``` diff --git a/deploy.yml b/deploy.yml new file mode 100644 index 0000000..50b6e34 --- /dev/null +++ b/deploy.yml @@ -0,0 +1,31 @@ +--- +- name: Setup environment + hosts: all + tasks: + - name: Load Default values + ansible.builtin.include_vars: + file: Default.yml + - name: Load OS specific values + ansible.builtin.include_vars: "{{ item }}" + with_first_found: + - files: + - "{{ ansible_os_family }}-{{ ansible_distribution_major_version }}.yml" + - "{{ ansible_os_family }}.yml" + - "{{ ansible_distribution }}.yml" + - "{{ ansible_system }}.yml" + skip: true + tags: + - always + +- name: Setup shared roles + hosts: all + roles: + - role: firewalld + +- name: Setup shared services + hosts: localhost + run_once: true + become: false + roles: + - role: redis-k8s + - role: elasticsearch-k8s diff --git a/files/k8s/k8s-cluster/manifests/fluent-bit.yaml b/files/k8s/k8s-cluster/manifests/fluent-bit.yaml new file mode 100644 index 0000000..20cff2f --- /dev/null +++ b/files/k8s/k8s-cluster/manifests/fluent-bit.yaml @@ -0,0 +1,186 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: fluent-bit + +--- +apiVersion: v1 +kind: Secret +metadata: + name: fluent-bit-auth + namespace: fluent-bit +type: Opaque +data: + username: dXNlcg== + password: eHh4LXh4eC14eHgteHh4LXh4eA== + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluent-bit + namespace: fluent-bit + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fluent-bit-read +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fluent-bit-read +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fluent-bit-read +subjects: + - kind: ServiceAccount + name: fluent-bit + namespace: fluent-bit + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluent-bit-config + namespace: fluent-bit +data: + fluent-bit.conf: | + [SERVICE] + Flush 1 + Log_Level error + Parsers_File parsers.conf + + [INPUT] + Name tail + Path /var/log/containers/*.log + Parser docker + Tag kube.* + Mem_Buf_Limit 5MB + Skip_Long_Lines On + + [INPUT] + Name systemd + Tag host.journald + Path /run/log/journal + DB /var/log/flb_journald.db + Read_From_Tail On + + [FILTER] + Name kubernetes + Match kube.* + Kube_URL https://kubernetes.default.svc:443 + Merge_Log On + K8S-Logging.Exclude Off + Labels On + Annotations On + + [FILTER] + Name grep + Match kube.* + Exclude container_name ingress-nginx + + [FILTER] + Name modify + Match * + Add node ${NODE_NAME} + + [FILTER] + Name modify + Match host.journald + Add log_type host + Add environment production + + [OUTPUT] + Name http + Match * + Host logs.clearwater.20c.dev + Port 443 + URI /insert/jsonline?_stream_fields=stream,node&_msg_field=log&_time_field=date + Format json_lines + json_date_format iso8601 + HTTP_User ${FLUENTBIT_USER} + HTTP_Passwd ${FLUENTBIT_PASSWORD} + tls On + + parsers.conf: | + [PARSER] + Name docker + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep On + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluent-bit + namespace: fluent-bit + labels: + k8s-app: fluent-bit +spec: + selector: + matchLabels: + k8s-app: fluent-bit + template: + metadata: + labels: + k8s-app: fluent-bit + spec: + serviceAccountName: fluent-bit + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + effect: "NoExecute" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: fluent-bit + image: fluent/fluent-bit:latest + volumeMounts: + - name: varlog + mountPath: /var/log + - name: journal + mountPath: /run/log/journal + - name: config + mountPath: /fluent-bit/etc/ + env: + - name: FLUENTBIT_USER + valueFrom: + secretKeyRef: + name: fluent-bit-auth + key: username + - name: FLUENTBIT_PASSWORD + valueFrom: + secretKeyRef: + name: fluent-bit-auth + key: password + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + hostAliases: + - ip: "10.31.101" + hostnames: + - "logs.somehost" + terminationGracePeriodSeconds: 10 + volumes: + - name: varlog + hostPath: + path: /var/log + - name: journal + hostPath: + path: /run/log/journal + - name: config + configMap: + name: fluent-bit-config diff --git a/files/k8s/k8s-cluster/manifests/metallb-config.yaml b/files/k8s/k8s-cluster/manifests/metallb-config.yaml new file mode 100644 index 0000000..41e952c --- /dev/null +++ b/files/k8s/k8s-cluster/manifests/metallb-config.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: public + namespace: metallb-system +spec: + addresses: + - x.x.x.x/32 + +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: external + namespace: metallb-system +spec: + ipAddressPools: + - public diff --git a/files/manifests/external-dns.yaml b/files/manifests/external-dns.yaml new file mode 100644 index 0000000..04c9b94 --- /dev/null +++ b/files/manifests/external-dns.yaml @@ -0,0 +1,67 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: external-dns + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: external-dns +rules: +- apiGroups: [""] + resources: ["services","endpoints","pods"] + verbs: ["get","watch","list"] +- apiGroups: ["extensions","networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get","watch","list"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["list"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: external-dns-viewer +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-dns +subjects: +- kind: ServiceAccount + name: external-dns + namespace: default + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: external-dns +spec: + replicas: 1 + selector: + matchLabels: + app: external-dns + strategy: + type: Recreate + template: + metadata: + labels: + app: external-dns + spec: + serviceAccountName: external-dns + containers: + - name: external-dns + image: registry.k8s.io/external-dns/external-dns:v0.14.2 + args: + - --source=ingress + - --domain-filter=apps.vaasl.io + - --provider=digitalocean + env: + - name: DO_TOKEN + valueFrom: + secretKeyRef: + name: digitalocean-dns + key: access-token diff --git a/files/manifests/secrets.yaml b/files/manifests/secrets.yaml new file mode 100644 index 0000000..aa51eb0 --- /dev/null +++ b/files/manifests/secrets.yaml @@ -0,0 +1,15 @@ +# This should be encrypted as a vault +--- +apiVersion: v1 +kind: Namespace +metadata: + name: cert-manager + +--- +apiVersion: v1 +data: + access-token: xxx +kind: Secret +metadata: + name: digitalocean-dns + namespace: cert-manager diff --git a/files/manifests/test-deployment.yaml b/files/manifests/test-deployment.yaml new file mode 100644 index 0000000..aefd6fd --- /dev/null +++ b/files/manifests/test-deployment.yaml @@ -0,0 +1,63 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx +spec: + replicas: 2 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:latest + ports: + - containerPort: 80 + +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-service + labels: + app: nginx +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: 80 + selector: + app: nginx + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: nginx-ingress + namespace: default + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" +spec: + ingressClassName: nginx + tls: + - hosts: + - x.ebenezer.20c.dev + secretName: nginx-tls + rules: + - host: x.ebenezer.20c.dev + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: nginx-service + port: + number: 80 \ No newline at end of file diff --git a/files/manifests/wildcard-vaasl-apps.yaml b/files/manifests/wildcard-vaasl-apps.yaml new file mode 100644 index 0000000..9971008 --- /dev/null +++ b/files/manifests/wildcard-vaasl-apps.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: wildcard-vaasl-apps + namespace: default +spec: + secretName: wildcard-vaasl-apps + issuerRef: + name: letsencrypt-prod-wild + kind: ClusterIssuer + group: cert-manager.io + commonName: "*.apps.vaasl.io" + dnsNames: + - "apps.vaasl.io" + - "*.apps.vaasl.io" diff --git a/galaxy-roles/requirements.yml b/galaxy-roles/requirements.yml new file mode 100644 index 0000000..50df5a0 --- /dev/null +++ b/galaxy-roles/requirements.yml @@ -0,0 +1,20 @@ +--- +- name: firewalld + scm: git + src: git@github.com:srwadleigh/ansible-role-firewalld.git + version: main + +- name: k8s + scm: git + src: git@github.com:srwadleigh/ansible-role-k8s.git + version: main + +- name: redis-k8s + scm: git + src: git@git.vdb.to:ops/redis-k8s.git + version: main + +- name: elasticsearch-k8s + scm: git + src: git@git.vdb.to:ops/elasticsearch-k8s.git + version: main diff --git a/group_vars/all/k8s_cluster/firewalld.yaml b/group_vars/all/k8s_cluster/firewalld.yaml new file mode 100644 index 0000000..f026be9 --- /dev/null +++ b/group_vars/all/k8s_cluster/firewalld.yaml @@ -0,0 +1,23 @@ +--- +firewalld_add: + - name: internal + services: + - ssh + - http + - https + - redis + - elasticsearch + ports: + - 6443/tcp + - 9100/tcp + + - name: public + services: + - http + - https + +firewalld_remove: + - name: public + services: + - dhcpv6-client + - ssh diff --git a/group_vars/all/k8s_cluster/main.yaml b/group_vars/all/k8s_cluster/main.yaml new file mode 100644 index 0000000..2be8b2d --- /dev/null +++ b/group_vars/all/k8s_cluster/main.yaml @@ -0,0 +1,94 @@ +--- +# K8S CLUSTER +k8s_type: rke2 +k8s_cluster_name: k8s-cluster +k8s_cluster_url: control1.somehost + +k8s_taint_servers: true +k8s_label_agents: true + +k8s_cni_manifest_override: true +k8s_cni_custom_template: canal-config.yaml +k8s_cni_interface: eth0 + +k8s_certmanager_version: v1.18.2 +k8s_metallb_version: v0.15.2 + +k8s_manifests: + - name: local-path-storage + type: template + + # RKE2 Ingress as a LoadBalancer + - name: rke2-ingress-loadbalancer + type: file + + # VIP LoadBalancer + - name: metallb + type: url + source: "{{ k8s_metallb_manifest_url }}" + + # TLS + - name: cert-manager + type: url + source: "{{ k8s_certmanager_manifest_url }}" + + - name: letsencrypt-prod + type: template + source: "{{ k8s_clusterissuer_template }}" + server: "{{ k8s_clusterissuer_le_prod_url }}" + solvers: + - type: http + ingress: nginx + +# example using cert-manager helm chart to insert custom values +# k8s_charts: +# - name: "{{ k8s_certmanager_name }}" +# namespace: "{{ k8s_certmanager_name }}" +# chart_ref: "{{ k8s_certmanager_chart }}" +# chart_version: v1.18.2 +# chart_values: +# crds: +# enabled: true +# config: +# featureGates: +# # Disable the use of Exact PathType in Ingress resources, to work around a bug in ingress-nginx +# # https://github.com/kubernetes/ingress-nginx/issues/11176 +# ACMEHTTP01IngressPathTypeExact: false +# extraEnv: +# - { name: HTTP_PROXY, value: "{{ k8s_http_proxy }}" } +# - { name: HTTPS_PROXY, value: "{{ k8s_https_proxy }}" } +# - { name: NO_PROXY, value: "{{ k8s_no_proxy }}" } +# cainjector: +# extraEnv: +# - { name: HTTP_PROXY, value: "{{ k8s_http_proxy }}" } +# - { name: HTTPS_PROXY, value: "{{ k8s_https_proxy }}" } +# - { name: NO_PROXY, value: "{{ k8s_no_proxy }}" } +# webhook: +# extraEnv: +# - { name: HTTP_PROXY, value: "{{ k8s_http_proxy }}" } +# - { name: HTTPS_PROXY, value: "{{ k8s_https_proxy }}" } +# - { name: NO_PROXY, value: "{{ k8s_no_proxy }}" } + +# REDIS +redis_namespace: default +redis_image: docker.io/redis +redis_image_tag: latest + +redis_sentinel_master: peeringdb-redis +redis_replicas: 3 +redis_min_available: 2 + +# ELASTICSEARCH +elasticsearch_namespace: default +elasticsearch_image: docker.elastic.co/elasticsearch/elasticsearch +elasticsearch_image_tag: 9.1.4 + +elasticsearch_cluster_name: es-cluster +elasticsearch_replicas: 3 +elasticsearch_min_available: 2 + +# requiredDuringSchedulingIgnoredDuringExecution,preferredDuringSchedulingIgnoredDuringExecution +elasticsearch_anti_affinity_type: preferredDuringSchedulingIgnoredDuringExecution + +# DoNotSchedule, ScheduleAnyway +elasticsearch_topology_constraint: ScheduleAnyway diff --git a/group_vars/all/k8s_cluster/vault.yaml b/group_vars/all/k8s_cluster/vault.yaml new file mode 100644 index 0000000..48384fa --- /dev/null +++ b/group_vars/all/k8s_cluster/vault.yaml @@ -0,0 +1,25 @@ +# This should be encrypted as a vault +--- +k8s_cluster_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +elasticsearch_secrets: + ELASTIC_PASSWORD: xxx-xxx-xxx-xxx-xxx-xxx + +redis_secrets: + REDIS_PASSWORD: xxx-xxx-xxx-xxx-xxx-xxx + +# cluster secrects could be defined here also +# k8s_secrets: +# - name: digitalocean-dns +# type: template +# namespace: cert-manager +# secrets: +# - key: access-token +# value: XXX +# - key: auth +# value: XXX +# +# - name: somes-secret +# type: file +# source: some-secret.yaml + diff --git a/hosts b/hosts new file mode 100644 index 0000000..2b0f5ad --- /dev/null +++ b/hosts @@ -0,0 +1,8 @@ +[k8s_cluster] +control1 k8s_node_type=bootstrap +agent1 k8s_node_type=agent k8s_pod_limit=1024 +agent2 k8s_node_type=agent k8s_pod_limit=1024 +agent3 k8s_node_type=agent k8s_pod_limit=1024 + +[k8s:children] +k8s_cluster