feat: replace longhorn with rook ceph

ansible/deploy_infra.yml

@@ -57,11 +57,12 @@
         traefik_namespace: ingress-internal
         traefik_default_ingress_class: true
         traefik_ingress_class_name: ingress-internal
-    - longhorn
+    - rook_operator
+    - rook_cluster
     - vault
     - cert_manager
-    - cert_manager_scaleway
     - trust_manager
+    - cert_manager_scaleway
     - role: external_dns
     - role: external_dns
       vars:

ansible/group_vars/kubernetes_master/all.yml

@@ -30,6 +30,8 @@ metallb_ip_address_pools:
       - 192.168.10.100-192.168.10.120
       - 2a0c:b641:02c0:110::100-2a0c:b641:02c0:110::120
     auto_assign: true
+rook_operator_localhost_kubeconfig_path: "{{ kubernetes_localhost_kubeconfig_path }}"
+rook_cluster_localhost_kubeconfig_path: "{{ kubernetes_localhost_kubeconfig_path }}"
 vault_localhost_kubeconfig_path: "{{ kubernetes_localhost_kubeconfig_path }}"
 vault_init_output_file_path: "{{ root_dir_role_path }}/vault.json.secrets"
 vault_ca_filename: "{{ ca_certificates_local_path_ca_certificate }}"

ansible/roles/cert_manager/tasks/main.yml

@@ -15,7 +15,7 @@
 
 - name: Download cert-manager crds"
   ansible.builtin.get_url:
-    url: "https://github.com/cert-manager/cert-manager/releases/download/{{ cert_manager_version }}/cert-manager.crds.yaml"
+    url: "https://github.com/cert-manager/cert-manager/releases/download/{{ cert_manager_chart_version }}/cert-manager.crds.yaml"
     dest: /tmp/cert_manager_crds.yaml
     owner: "{{ ansible_user }}"
     group: "{{ ansible_user }}"

ansible/roles/common_kubernetes/tasks/main.yml

@@ -8,22 +8,3 @@
     path: /etc/fstab
     regexp: '^([^#].*?\sswap\s+sw\s+.*)$'
     replace: '# \1'
-
-- name: Install required packages by longhorn
-  ansible.builtin.apt:
-    pkg:
-      - open-iscsi
-      - nfs-common
-    state: present
-
-- name: Enable and start iscsid required by longhorn
-  ansible.builtin.systemd:
-    name: iscsid
-    enabled: true
-    state: started
-
-- name: Disable and stop multipathd required by longhorn
-  ansible.builtin.systemd:
-    name: multipathd
-    enabled: false
-    state: stopped

ansible/roles/k3s/templates/config.yaml.j2

@@ -25,7 +25,7 @@ flannel-backend: none
 disable-network-policy: true
 disable-kube-proxy: true
 
-disable: traefik,servicelb,metrics-server
+disable: traefik,servicelb
 
 write-kubeconfig-mode: 600
 write-kubeconfig: /etc/rancher/k3s/kubeconfig

ansible/roles/rook_cluster/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+# datasource=github-tags depName=rook/rook
+rook_cluster_chart_version: "v1.12.8"
+rook_cluster_force_install: false
+rook_cluster_localhost_kubeconfig_path: /home/user/.kube/kubeconfig

ansible/roles/longhorn/meta/main.yml → ansible/roles/rook_cluster/meta/main.yml

@@ -1,8 +1,8 @@
 ---
 galaxy_info:
-  role_name: longhorn
+  role_name: rook_cluster
   author: Ludovic Ortega
-  description: Install and configure longhorn
+  description: Install and configure rook cluster
   license: CeCILL
   min_ansible_version: "2.16.0"
   platforms:

ansible/roles/rook_cluster/tasks/main.yml

@@ -0,0 +1,37 @@
+---
+- name: Gather information of rook-ceph chart
+  kubernetes.core.helm_info:
+    name: rook-ceph
+    release_namespace: rook-ceph-cluster
+    kubeconfig: "{{ rook_cluster_localhost_kubeconfig_path }}"
+    validate_certs: true
+  register: rook_cluster_info
+  delegate_to: localhost
+
+- name: Check if rook is already installed
+  ansible.builtin.set_fact:
+    rook_cluster_is_installed: "{% if 'status' in rook_cluster_info %}true{% else %}false{% endif %}"
+  delegate_to: localhost
+
+- name: Add rook-release helm repo
+  kubernetes.core.helm_repository:
+    name: rook-release
+    repo_url: https://charts.rook.io/release
+    kubeconfig: "{{ rook_cluster_localhost_kubeconfig_path }}"
+    validate_certs: true
+  delegate_to: localhost
+
+- name: Deploy rook-ceph cluster chart
+  kubernetes.core.helm:
+    name: rook-ceph
+    chart_ref: rook-release/rook-ceph-cluster
+    chart_version: "{{ rook_cluster_chart_version[1:] }}"
+    release_namespace: rook-ceph-cluster
+    create_namespace: true
+    release_values: "{{ lookup('ansible.builtin.template', 'helm-values.yml.j2') | from_yaml }}"
+    update_repo_cache: true
+    wait: true
+    kubeconfig: "{{ rook_cluster_localhost_kubeconfig_path }}"
+    validate_certs: true
+  delegate_to: localhost
+  when: not rook_cluster_is_installed or rook_cluster_force_install

ansible/roles/rook_cluster/templates/helm-values.yml.j2

@@ -0,0 +1,76 @@
+cephClusterSpec:
+  storage:
+    deviceFilter: sdb
+  mgr:
+    modules:
+      - name: pg_autoscaler
+        enabled: true
+      - name: rook
+        enabled: true
+  dashboard:
+    enabled: true
+    ssl: false
+cephObjectStores:
+  - name: ceph-objectstore
+    # see https://github.com/rook/rook/blob/master/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md#object-store-settings for available configuration
+    spec:
+      metadataPool:
+        failureDomain: host
+        replicated:
+          size: 3
+      dataPool:
+        failureDomain: host
+        erasureCoded:
+          dataChunks: 2
+          codingChunks: 1
+      preservePoolsOnDelete: true
+      gateway:
+        port: 80
+        resources:
+          limits:
+            cpu: "2000m"
+            memory: "2Gi"
+          requests:
+            cpu: "1000m"
+            memory: "1Gi"
+        # securePort: 443
+        # sslCertificateRef:
+        instances: 1
+        priorityClassName: system-cluster-critical
+    storageClass:
+      enabled: true
+      name: ceph-bucket
+      reclaimPolicy: Delete
+      volumeBindingMode: "Immediate"
+      # see https://github.com/rook/rook/blob/master/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md#storageclass for available configuration
+      parameters:
+        # note: objectStoreNamespace and objectStoreName are configured by the chart
+        region: eu-west-1
+    ingress:
+      enabled: true
+      annotations:
+        cert-manager.io/cluster-issuer: "vault"
+        cert-manager.io/private-key-algorithm: "RSA"
+        cert-manager.io/private-key-size: "4096"
+      host:
+        name: s3.unicornafk.fr
+        path: /
+      tls:
+        - secretName: s3.unicornafk.fr
+          hosts:
+            - s3.unicornafk.fr
+      ingressClassName: ingress-internal
+ingress:
+  dashboard:
+    annotations:
+      cert-manager.io/cluster-issuer: "vault"
+      cert-manager.io/private-key-algorithm: "RSA"
+      cert-manager.io/private-key-size: "4096"
+    host:
+      name: ceph.unicornafk.fr
+      path: /
+    tls:
+      - secretName: ceph.unicornafk.fr
+        hosts:
+          - ceph.unicornafk.fr
+    ingressClassName: ingress-internal

ansible/roles/rook_operator/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+# datasource=github-tags depName=rook/rook
+rook_operator_chart_version: "v1.12.8"
+rook_operator_force_install: false
+rook_operator_localhost_kubeconfig_path: /home/user/.kube/kubeconfig

ansible/roles/rook_operator/meta/main.yml

@@ -0,0 +1,11 @@
+---
+galaxy_info:
+  role_name: rook_operator
+  author: Ludovic Ortega
+  description: Install and configure rook operator
+  license: CeCILL
+  min_ansible_version: "2.16.0"
+  platforms:
+    - name: Ubuntu
+      versions:
+        - jammy

ansible/roles/rook_operator/tasks/main.yml

@@ -0,0 +1,36 @@
+---
+- name: Gather information of rook-ceph operator chart
+  kubernetes.core.helm_info:
+    name: rook-ceph
+    release_namespace: rook-ceph
+    kubeconfig: "{{ rook_operator_localhost_kubeconfig_path }}"
+    validate_certs: true
+  register: rook_operator_info
+  delegate_to: localhost
+
+- name: Check if rook operator is already installed
+  ansible.builtin.set_fact:
+    rook_operator_is_installed: "{% if 'status' in rook_operator_info %}true{% else %}false{% endif %}"
+  delegate_to: localhost
+
+- name: Add rook-release helm repo
+  kubernetes.core.helm_repository:
+    name: rook-release
+    repo_url: https://charts.rook.io/release
+    kubeconfig: "{{ rook_operator_localhost_kubeconfig_path }}"
+    validate_certs: true
+  delegate_to: localhost
+
+- name: Deploy rook-ceph operator chart
+  kubernetes.core.helm:
+    name: rook-ceph
+    chart_ref: rook-release/rook-ceph
+    chart_version: "{{ rook_operator_chart_version[1:] }}"
+    release_namespace: rook-ceph
+    create_namespace: true
+    update_repo_cache: true
+    wait: true
+    kubeconfig: "{{ rook_operator_localhost_kubeconfig_path }}"
+    validate_certs: true
+  delegate_to: localhost
+  when: not rook_operator_is_installed or rook_operator_force_install

ansible/roles/vault/templates/helm-values.yml.j2

@@ -42,14 +42,6 @@ server:
   service:
     ipFamilyPolicy: "PreferDualStack"
 
-  resources:
-    requests:
-      memory: 4Gi
-      cpu: 1000m
-    limits:
-      memory: 8Gi
-      cpu: 2000m
-
   readinessProbe:
     httpGet:
       scheme: HTTPS

argocd/keycloak/manifests/03_keycloak.yaml

@@ -116,9 +116,6 @@ spec:
               limits:
                 memory: 2Gi
                 cpu: "1"
-              requests:
-                memory: 256Mi
-                cpu: "0.2"
       volumes:
         - name: keycloak-vault
           secret: