feat: Enable CEX based LUKS encryption

    1. Configure the CEX in KVM host
    2. Use the right device type ignition for MCO
    3. Attach the CEX mediated device to guest vm

Signed-off-by: Madhu Pillai <madhupillai80@gmail.com>

Author: madhu-pillai

docs/set-variables-group-vars.md

@@ -12,6 +12,8 @@
 :--- | :--- | :---
 **installation_type** | Can be of type kvm or lpar. Some packages will be ignored for installation in case of non lpar based installation. | kvm 
 **controller_sudo_pass** | The password to the machine running Ansible (localhost). This will only be used for two things. To ensure you've installed the pre-requisite packages if you're on Linux, and to add the login URL to your /etc/hosts file. | Pas$w0rd!
+**cex** | Whether to enable cex based luks encryption, default to False.
+**luks_device** | Specify the storage device type used for LUKS encryption. This setting determines which MCO Ignition configuration will be applied. Use in combination with the cex parameter.  [dasd, fcp, virt]
 
 ## 2 - LPAR(s)
 **Variable Name** | **Description** | **Example**
@@ -123,6 +125,7 @@
 **env.cluster.nodes.control.ipv6** | IPv6 address for the control nodes. Use iprovided list formatting (if use_ipv6 variable is 'True'). | fd00::5fd00::6fd00::7
 **env.cluster.nodes.control.mac** | MAC address for the control node if use_dhcp variable is 'True'. | 52:54:00:18:1A:2B
 **env.cluster.nodes.control.hostname** | Hostnames for control nodes. Must match the total number of IP addresses for control nodes (usually 3). If DNS is hosted on the bastion, this can be anything. If DNS is hosted elsewhere, this must match DNS definition. This will be combined with the metadata_name and base_domain to create a Fully Qualififed Domain Name (FQDN). | control-01control-02control-03
+**env.cluster.nodes.control.cex_card** | Configure the CCA Crypto Express (CEX) card and assign the domain number to correspond with the control hostname specified in the environment configuration. The UUID's are hardcoded. Add more as needed using the format: UUID:00.0028
 
 ## 9 - Compute Nodes
 **Variable Name** | **Description** | **Example**
@@ -136,6 +139,7 @@
 **env.cluster.nodes.control.ipv6** | IPv6 address for the compute nodes. Use iprovided list formatting (if use_ipv6 variable is 'True'). | fd00::8fd00::9
 **env.cluster.nodes.compute.mac** | MAC address for the compute node if use_dhcp variable is 'True'. | 52:54:00:18:1A:2B
 **env.cluster.nodes.compute.hostname** | Hostnames for compute nodes. Must match the total number of IP addresses and VM names for compute nodes. If DNS is hosted on the bastion, this can be anything. If DNS is hosted elsewhere, this must match DNS definition. This will be combined with the metadata_name and base_domain to create a Fully Qualififed Domain Name (FQDN). | compute-01compute-02
+**env.cluster.nodes.compute.cex_card** | Configure the CCA Crypto Express (CEX) card and assign the domain number to correspond with the compute hostname specified in the environment configuration. The UUID's are hardcoded. Add more as needed using the format: UUID:00.0028
 
 ## 10 - Infra Nodes
 **Variable Name** | **Description** | **Example**

inventories/default/group_vars/all.yaml.template

@@ -6,6 +6,8 @@
 # Section 1 - Ansible Controller
 installation_type: kvm
 controller_sudo_pass: #X
+cex: False
+luks_device: #[dasd | fcp | virt ] 
 
 env:
 
@@ -140,6 +142,10 @@ env:
 #         - X
 #         - X
 #         - X
+        cex_card:
+          - 68cd2d83-3eef-4e45-b22c-534f90b16cb9:xx.xxxx #xx.xxxx means the CEX cca card number from lszcrypt.
+          - ba3f873c-3022-4534-9072-eb0d4530b137:xx.xxxx
+          - 610e4724-c8c1-45ed-a498-ccfdadc1f2ca:xx.xxxx
 
 # Section 9 - Compute Nodes
       compute:
@@ -162,6 +168,9 @@ env:
         hostname:
 #         - X
 #         - X
+        cex_card:
+          - 5c84eefb-cb45-4519-86d3-ba23e65e8896:xx.xxxx #xx.xxxx means the CEX cca card number from lszcrypt
+          - 60cf6ec4-2741-44e2-84d2-0746f133db16:xx.xxxx
 
 # Section 10 - Infra Nodes
 #      infra:

playbooks/3_setup_kvm_host.yaml

@@ -188,3 +188,9 @@
   roles:
     - configure_storage
     - { role: macvtap, when: env.network_mode | upper != 'NAT' }
+
+- hosts: kvm_host
+  tags: setup, section_3
+  become: true
+  roles:
+    - { role: configure_cex, when: cex | default(false) | bool } 

playbooks/create_provision_cex_nodes.yaml

@@ -0,0 +1,102 @@
+---
+
+- name: CEX create nodes - find LPAR host files
+  hosts: localhost
+  vars_files:
+    - "{{ inventory_dir }}/group_vars/all.yaml"
+  tasks:
+    - block:
+        - name: Loop over node types and include role
+          include_role:
+            name: check_for_lpar_nodes
+          loop:
+            - bootstrap
+            - control
+            - compute
+          loop_control:
+            loop_var: node_type
+          register: result
+      ignore_errors: true
+
+    - name: Fail the play if the previous command did not succeed
+      fail:
+        msg: "Check for LPAR nodes failed for bootstrap or control node"
+      when:
+        - "'bootstrap' in result.msg"
+        - "'control' in result.msg"
+
+- name: CEX create nodes - prepare KVM guests
+  hosts: kvm_host
+  gather_facts: false
+  vars_files:
+    - "{{ inventory_dir }}/group_vars/all.yaml"
+  roles:
+    - { role: prep_kvm_guests, when: env.cluster.nodes.bootstrap.vm_name not in hosts_with_host_vars }
+    - { role: delete_nodes, when: env.cluster.nodes.bootstrap.vm_name not in hosts_with_host_vars }
+
+- name: CEX create nodes - create bootstrap
+  hosts: kvm_host[0]
+  gather_facts: false
+  vars_files:
+    - "{{ inventory_dir }}/group_vars/all.yaml"
+  roles:
+    - { role: common, when: env.cluster.nodes.bootstrap.vm_name not in hosts_with_host_vars }
+    - { role: create_bootstrap, when: env.cluster.nodes.bootstrap.vm_name not in hosts_with_host_vars }
+
+# CONTROL NODE CREATION
+- name: CEX create nodes - create control nodes
+  hosts: kvm_host
+  gather_facts: false
+  vars_files:
+    - "{{ inventory_dir }}/group_vars/all.yaml"
+  roles:
+    - role: common
+    - role: create_cex_nodes
+      vars:
+        create_phase: "control"
+      when: cex | bool
+
+- name: CEX create nodes - wait for bootstrap to connect control plane
+  hosts: bastion
+  become: true
+  gather_facts: true
+  vars_files:
+    - "{{ inventory_dir }}/group_vars/all.yaml"
+  environment:
+    KUBECONFIG: "{{ '/home/' if (env.bastion.access.user != 'root') else '/'}}{{ env.bastion.access.user }}/.kube/config"
+  roles:
+    - wait_for_bootstrap
+
+- name: CEX create nodes - tear down bootstrap
+  hosts: kvm_host[0]
+  gather_facts: false
+  tags: teardown_bootstrap
+  vars_files:
+    - "{{ inventory_dir }}/group_vars/all.yaml"
+  tasks:
+    - name: Destroy bootstrap VM
+      community.libvirt.virt:
+        name: "{{ env.cluster.nodes.bootstrap.vm_name }}"
+        command: destroy
+      ignore_errors: true
+      when: env.cluster.nodes.bootstrap.vm_name not in hosts_with_host_vars
+
+    - name: Undefine bootstrap VM
+      community.libvirt.virt:
+        name: "{{ env.cluster.nodes.bootstrap.vm_name }}"
+        command: undefine
+      ignore_errors: true
+
+# COMPUTE NODE CREATION
+- name: CEX create nodes - create compute nodes
+  hosts: kvm_host
+  gather_facts: false
+  tags: create_cex_nodes
+  vars_files:
+    - "{{ inventory_dir }}/group_vars/all.yaml"
+  roles:
+    - role: common
+    - role: create_cex_nodes
+      vars:
+        create_phase: "compute"
+      when: env.cluster.nodes.bootstrap.vm_name not in hosts_with_host_vars

roles/configure_cex/tasks/main.yaml

@@ -0,0 +1,39 @@
+- name: Load vfio_ap kernel module
+  ansible.builtin.modprobe:
+    name: vfio_ap
+    state: present
+
+- name: Set defaults AP masks
+  ansible.builtin.shell: |
+    echo 0x0 > /sys/bus/ap/apmask
+    echo 0x0 > /sys/bus/ap/aqmask
+  args:
+    executable: /bin/bash
+
+- name: Combine all CEX card assignments from control and compute nodes
+  set_fact:
+    cex_cards: "{{ (env.cluster.nodes.control.cex_card | default([])) + (env.cluster.nodes.compute.cex_card | default([])) }}"
+
+- name: Deduplicate CEX cards list (in case of shared UUIDs)
+  set_fact:
+    cex_cards: "{{ cex_cards | unique }}"
+
+- name: Debug final list of CEX UUID assignments
+  debug:
+    var: cex_cards
+
+- name: Create VFIO assignment script for all CEX cards
+  template:
+    src: assign_cards.sh.j2
+    dest: /tmp/assign_all_cex_cards.sh
+    mode: '0755'
+
+- name: Execute VFIO assignment script
+  shell: /tmp/assign_all_cex_cards.sh
+  args:
+    executable: /bin/bash
+
+- name: Housekeep temporary assignment script
+  file:
+    path: /tmp/assign_all_cex_cards.sh
+    state: absent

roles/configure_cex/templates/assign_cards.sh.j2

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Reference document for the cex configuration in zKVM
+# https://www.ibm.com/docs/en/linux-on-systems?topic=management-configuring-crypto-express-adapters-kvm-guests
+
+# Configure each CEX card
+{% for entry in cex_cards %}
+{% set uuid = entry.split(':')[0] %}
+{% set matrix_val = entry.split(':')[1] %}
+{% set adapter = matrix_val.split('.')[0] %}
+{% set domain = matrix_val.split('.')[1] %}
+
+uuid="{{ uuid }}"
+matrix_val="{{ matrix_val }}"
+adapter="{{ adapter }}"
+domain="{{ domain }}"
+
+uuid_path="/sys/devices/vfio_ap/matrix/$uuid"
+matrix_file="$uuid_path/matrix"
+
+if [ -d "$uuid_path" ]; then
+    if grep -q "{{ matrix_val }}" "$matrix_file" 2>/dev/null; then
+        echo "[INFO] UUID $uuid already configured with matrix {{ matrix_val }} — skipping."
+    else
+        echo "[WARN] UUID $uuid exists, but matrix entry '{{ matrix_val }}' not found!"
+        echo "[WARN] Please reboot the node and try again."
+        exit 1
+    fi
+else
+    echo "[INFO] Creating UUID $uuid with adapter $adapter and domain $domain"
+    echo "$uuid" > /sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/create
+    echo "0x$adapter" > "$uuid_path/assign_adapter"
+    echo "0x$domain" > "$uuid_path/assign_domain"
+fi
+
+{% endfor %}

roles/create_cex_nodes/tasks/attach_cex_device.yaml

@@ -0,0 +1,24 @@
+- name: Extract UUID from cex_pair
+  set_fact:
+    cex_uuid: "{{ cex_pair.split(':')[0] }}"
+    cex_bdf: "{{ cex_pair.split(':')[1] }}"
+
+- name: Debug CEX assignment
+  debug:
+    msg: "Attaching CEX UUID {{ cex_uuid }} to VM {{ vm_name }}"
+
+- name: Generate hostdev XML from template
+  template:
+    src: hostdev.xml.j2
+    dest: "/tmp/cex_{{ vm_name }}_{{ cex_uuid }}.xml"
+
+- name: Check if CEX UUID already attached
+  shell: |
+    virsh dumpxml {{ vm_name }} | grep -q "{{ cex_uuid }}"
+  register: cex_attached
+  ignore_errors: true
+
+- name: Attach CEX device if not already attached
+  shell: |
+    virsh attach-device {{ vm_name }} /tmp/cex_{{ vm_name }}_{{ cex_uuid }}.xml --persistent
+  when: cex_attached.rc != 0

roles/create_cex_nodes/tasks/create_compute_node_loop.yaml

@@ -0,0 +1,6 @@
+- name: Loop over compute node names and call per-node logic
+  include_tasks: create_compute_node_single.yaml
+  loop: "{{ env.cluster.nodes.compute.vm_name }}"
+  loop_control:
+    loop_var: compute_node_name
+    index_var: control_index

roles/create_cex_nodes/tasks/create_compute_node_single.yaml

@@ -0,0 +1,58 @@
+- name: Set facts for current compute node
+  set_fact:
+    vm_name: "{{ compute_node_name }}"
+    vm_hostname: "{{ env.cluster.nodes.compute.hostname[control_index] }}"
+    vm_mac: "{{ env.cluster.nodes.compute.mac[control_index] | default('') }}"
+    vm_ip: "{{ env.cluster.nodes.compute.ip[control_index] }}"
+    vm_ipv6: "{{ env.cluster.nodes.compute.ipv6[control_index] | default('') }}"
+
+- name: Check if VM {{ vm_name }} already exists
+  command: "virsh dominfo {{ vm_name }}"
+  register: vm_check
+  failed_when: false
+  changed_when: false
+
+- name: Skip creation of VM {{ vm_name }} if it already exists
+  debug:
+    msg: "VM {{ vm_name }} already exists, skipping creation."
+  when: vm_check.rc == 0
+
+- name: Create CoreOS compute node VM {{ vm_name }}
+  shell: |
+    virt-install \
+      --name {{ vm_name }} \
+      --osinfo detect=on,name={{ ('rhel8.6') if rhcos_os_variant is not defined else (rhcos_os_variant) }} \
+      --disk pool={{ env.cluster.networking.metadata_name }}-vdisk,size={{ env.cluster.nodes.compute.disk_size }},cache=none,io=native \
+      --ram {{ env.cluster.nodes.compute.ram }} \
+      {{ env.cluster.nodes.compute.vcpu_model_option | default('--cpu host') }} \
+      --vcpus {{ env.cluster.nodes.compute.vcpu }} \
+      --network network={{ env.vnet_name }}{% if vm_mac and env.use_dhcp %},mac={{ vm_mac }}{% endif %} \
+      --location {{ rhcos_download_url }},kernel={{ rhcos_live_kernel }},initrd={{ rhcos_live_initrd }} \
+      --extra-args "rd.neednet=1 coreos.inst=yes coreos.inst.install_dev=vda" \
+      --extra-args "coreos.live.rootfs_url=http://{{ env.bastion.networking.ip }}:8080/bin/{{ rhcos_live_rootfs }}" \
+      {% if env.use_dhcp and vm_mac %}
+      --extra-args "ip=dhcp" \
+      {% else %}
+      --extra-args "ip={{ vm_ip }}::{{ env.cluster.networking.gateway }}:{{ env.cluster.networking.subnetmask }}:{{ vm_hostname }}.{{ env.cluster.networking.metadata_name }}.{{ env.cluster.networking.base_domain }}:{{ env.cluster.networking.interface }}:none:1500" \
+      --extra-args "{{ ('ip=[' + vm_ipv6 + ']::[' + env.cluster.networking.ipv6_gateway +']:' + env.cluster.networking.ipv6_prefix | string + '::' + env.cluster.networking.interface + ':none' ) if env.use_ipv6 == True else '' }}" \
+      {% endif %}
+      --extra-args "nameserver={{ env.cluster.networking.nameserver1 }}{% if env.cluster.networking.nameserver2 is defined %},{{ env.cluster.networking.nameserver2 }}{% endif %}" \
+      --extra-args "coreos.inst.ignition_url=http://{{ env.bastion.networking.ip }}:8080/ignition/worker.ign" \
+      --extra-args "{{ _vm_console }}" \
+      --memballoon none \
+      --graphics none \
+      --console pty,target_type=serial \
+      --wait=-1 \
+      --noautoconsole
+  args:
+    executable: /bin/bash
+
+- name: Set CEX cards for {{ vm_name }}
+  set_fact:
+    cex_cards: ["{{ env.cluster.nodes.compute.cex_card[control_index] }}"]
+
+- name: Attach each CEX device from assignment list
+  include_tasks: attach_cex_device.yaml
+  loop: "{{ cex_cards }}"
+  loop_control:
+    loop_var: cex_pair

roles/create_cex_nodes/tasks/create_control_node_loop.yaml

@@ -0,0 +1,6 @@
+- name: Loop over control node names and call per-node logic
+  include_tasks: create_control_node_single.yaml
+  loop: "{{ env.cluster.nodes.control.vm_name }}"
+  loop_control:
+    loop_var: control_node_name
+    index_var: control_index