Add support to ZK for client-server and server-server authentication with mTLS

.gitignore

@@ -8,3 +8,4 @@ venv
 molecule/shared/certs/*
 !molecule/shared/certs/configs/
 !molecule/shared/certs/Makefile
+.vscode

molecule/cluster/molecule.yml

@@ -32,7 +32,6 @@ platforms:
     networks:
       - name: molecule
 
-
   - name: machine3
     image: registry.access.redhat.com/ubi8/ubi-init:latest
     pre_build_image: true
@@ -55,7 +54,6 @@ provisioner:
   inventory:
     group_vars:
       all:
-        kafka_home: /opt/kafka/kafka_2.13-3.1.1
         kafka_advertised_interface_name: eth0
         zookeeper_interface_name: eth0
 
@@ -71,8 +69,11 @@ provisioner:
 
         kafka_tls_config:
           enabled: true
-          trustedCA: /etc/certs/ca-root.pem
+          trustedCA:
+            file: ../../../molecule/shared/certs/ca-root.pem
+            location: /etc/certs/ca-root.pem
           keystore:
+            file: ../../../molecule/shared/certs/cert-cluster.p12
             location: /etc/certs/cert-cluster.p12
             password: changeit
             type: PKCS12
@@ -106,12 +107,17 @@ provisioner:
         kafka_admin:
           listener_name: admin
           authentication:
-            tls:
-              trustedCA: /etc/certs/ca-root.pem
-              keystore:
-                location: /etc/certs/cert-cluster.p12
-                password: changeit
-                type: PKCS12
+            type: tls
+          tls:
+            enabled: true
+            trustedCA:
+              file: ../../../molecule/shared/certs/ca-root.pem
+              location: /etc/certs/ca-root.pem
+            keystore:
+              file: ../../../molecule/shared/certs/cert-admin.p12
+              location: /etc/certs/cert-admin.p12
+              password: changeit
+              type: PKCS12
 
     host_vars:
       machine1:

molecule/zk-secured/converge.yml

@@ -0,0 +1,2 @@
+---
+- import_playbook: ../../playbooks/setup.yml

molecule/zk-secured/molecule.yml

@@ -0,0 +1,113 @@
+---
+# skip prerun because I am experiencing the issue described here:
+# https://github.com/ansible/ansible-lint/issues/2070
+prerun: false
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint: |
+  ansible-lint -v
+platforms:
+
+  - name: machine1
+    image: registry.access.redhat.com/ubi8/ubi-init:latest
+    pre_build_image: true
+    privileged: true
+    command: "/usr/sbin/init"
+    groups:
+      - kafka
+      - zookeeper
+    networks:
+      - name: molecule
+
+  - name: machine2
+    image: registry.access.redhat.com/ubi8/ubi-init:latest
+    pre_build_image: true
+    privileged: true
+    command: "/usr/sbin/init"
+    groups:
+      - kafka
+      - zookeeper
+    networks:
+      - name: molecule
+
+  - name: machine3
+    image: registry.access.redhat.com/ubi8/ubi-init:latest
+    pre_build_image: true
+    privileged: true
+    command: "/usr/sbin/init"
+    groups:
+      - kafka
+      - zookeeper
+    networks:
+      - name: molecule
+
+provisioner:
+  name: ansible
+  ansible_args:
+    - --diff
+  playbooks:
+    converge: converge.yml
+    prepare: ../shared/prepare.yml
+    verify: verify.yml
+  inventory:
+    group_vars:
+      zookeeper:
+        zookeeper_interface_name: eth0
+
+        zookeeper_auth_client_enabled: true
+        zookeeper_auth_quorum_enabled: true
+        zookeeper_auth_type: tls
+
+        zookeeper_tls_config:
+          enabled: true
+          trustedCA:
+            file: ../../../molecule/shared/certs/ca-root.pem
+            location: /etc/certs/ca-root.pem
+          keystore:
+            file: ../../../molecule/shared/certs/cert-cluster.p12
+            location: /etc/certs/cert-cluster.p12
+            password: changeit
+            type: PKCS12
+
+      kafka:
+        kafka_advertised_interface_name: eth0
+        kafka_advertised_host: "{{ ansible_hostname }}"
+
+        kafka_tls_config:
+          enabled: true
+          trustedCA:
+            file: ../../../molecule/shared/certs/ca-root.pem
+            location: /etc/certs/ca-root.pem
+          keystore:
+            file: ../../../molecule/shared/certs/cert-cluster.p12
+            location: /etc/certs/cert-cluster.p12
+            password: changeit
+            type: PKCS12
+
+        kafka_listeners:
+
+          - name: plain
+            port: 9092
+            tls: false
+
+        kafka_inter_broker_listener_name: plain
+
+        kafka_admin:
+          listener_name: plain
+
+    host_vars:
+      machine1:
+        kafka_broker_id: 1
+        zookeeper_myid: 1
+      machine2:
+        kafka_broker_id: 2
+        zookeeper_myid: 2
+      machine3:
+        kafka_broker_id: 3
+        zookeeper_myid: 3
+      localhost:
+        ansible_python_interpreter: "{{ ansible_playbook_python }}"
+verifier:
+  name: ansible

molecule/zk-secured/verify.yml

@@ -0,0 +1,47 @@
+---
+- name: Verify
+  hosts: all
+  gather_facts: true
+  vars: 
+    kafka_home: /opt/kafka/kafka_2.13-3.7.0
+  tasks:
+
+  - name: Ensure ZK CLI config file exist
+    ansible.builtin.copy:
+      content: |
+        zookeeper.ssl.keystore.location=/etc/certs/cert-cluster.p12
+        zookeeper.ssl.keystore.password=changeit
+        zookeeper.ssl.keystore.type=PKCS12
+        zookeeper.ssl.truststore.location=/etc/certs/ca-root.pem
+        zookeeper.ssl.truststore.type=PEM
+        zookeeper.ssl.client.enable=true
+        zookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty
+      dest: /tmp/zk-tls-config.properties
+
+  - name: Test ZK CLI mTLS connection
+    ansible.builtin.command: >
+      {{ kafka_home }}/bin/zookeeper-shell.sh \
+        {{ ansible_default_ipv4.address }}:2182 \
+        whoami \
+        -zk-tls-config-file /tmp/zk-tls-config.properties
+    changed_when: false
+    register: zk_cli_res
+
+  - name: ZK CLI mTLS connection assertions
+    ansible.builtin.assert:
+      that: 
+        - zk_cli_res.stdout_lines is search('SyncConnected') 
+        - zk_cli_res.stdout_lines is search('x509:\ CN=kafka')
+
+  - name: Test Kafka to ZK mTLS connection
+    ansible.builtin.command: >
+      {{ kafka_home }}/bin/kafka-topics.sh \
+        --bootstrap-server {{ ansible_default_ipv4.address }}:9092 \
+        --describe
+    changed_when: false
+    run_once: true
+    register: kafka_zk_res
+
+  - name: Kafka to ZK mTLS connection assertions
+    ansible.builtin.assert:
+      that: kafka_zk_res.rc == 0

plugins/action/kafka_config.py

@@ -17,13 +17,18 @@
     keystore=dict(
       type='dict',
       options=dict(
+        file=dict(type='str'),
         location=dict(type='str'),
         password=dict(type='str'),
         type=dict(type='str')
       )
     ),
     trustedCA=dict(
-      type='str'
+      type='dict',
+      options=dict(
+        file=dict(type='str'),
+        location=dict(type='str')
+      )
     )
   )
 )
@@ -110,7 +115,7 @@ def run(self, tmp=None, task_vars=None):
     ret = dict()
     ret['__kafka_config'] = kafka_config
     ret['__admin_config'] = generator.get_admin_config(kafka_config)
-    
+
 
     return dict(ansible_facts=dict(ret))
 
@@ -152,10 +157,8 @@ def _coalesce_listeners(self, listeners):
         listener['protocol'] = 'PLAINTEXT'
 
       if tls:
-        # TODO support different trustore types
         listener['truststore_type'] = 'PEM'
-        listener['truststore_location'] = tls['trustedCA']
-        # listener['truststore_password'] = 
+        listener['truststore_location'] = tls['trustedCA']['location']
 
         listener['keystore_location'] = tls['keystore']['location']
         listener['keystore_password'] = tls['keystore']['password']
@@ -246,10 +249,10 @@ def get_admin_config(self, kafka_config):
     options = {}
 
     authentication = admin.pop('authentication', {})
-    tls = authentication.pop('tls', {})
+    tls = admin.pop('tls', {})
 
     if 'trustedCA' in tls:
-      options['ssl.truststore.location']=tls['trustedCA']
+      options['ssl.truststore.location']=tls['trustedCA']['location']
       options['ssl.truststore.type']='PEM'
 
     if 'keystore' in tls:
@@ -291,7 +294,5 @@ def get_admin_config(self, kafka_config):
       'protocol': protocol,
       'listener': '%s:%s' % (listener['advertised'], listener['port']),
       'require_command_config': bool(options),
-      'options': options 
+      'options': options
     }
-
-

roles/kafka_systemd_broker/tasks/setup_admin.yml

@@ -10,4 +10,3 @@
     - __admin_config.require_command_config is defined
     - __admin_config.require_command_config
   become: true
-

roles/kafka_systemd_broker/tasks/setup_security.yml

@@ -4,7 +4,22 @@
     kafka_java_opts: "{{ kafka_java_opts + [
         '-Djava.security.auth.login.config=' + kafka_jaas_config_file_location
       ] }}"
-  when: zookeeper_auth_type is defined
+  when: 
+    - zookeeper_auth_client_enabled
+    - zookeeper_auth_type is defined
+    - zookeeper_auth_type == 'gssapi'
+
+- name: Set SSL facts
+  ansible.builtin.set_fact:
+    kafka_ssl_truststore_file: "{{ kafka_tls_config.trustedCA.file }}"
+    kafka_ssl_truststore_location: "{{ kafka_tls_config.trustedCA.location }}"
+    kafka_ssl_keystore_file: "{{ kafka_tls_config.keystore.file }}"
+    kafka_ssl_keystore_location: "{{ kafka_tls_config.keystore.location }}"
+    kafka_ssl_keystore_password: "{{ kafka_tls_config.keystore.password }}"
+    kafka_ssl_keystore_type: "{{ kafka_tls_config.keystore.type }}"
+  when: 
+  - kafka_tls_config is defined
+  - kafka_tls_config.enabled
 
 - name: Create security dir for Kafka
   ansible.builtin.file:
@@ -22,20 +37,23 @@
     owner: "{{ kafka_user }}"
     group: "{{ kafka_group }}"
     mode: 0400
-  when: zookeeper_auth_type is defined
+  when: 
+    - zookeeper_auth_client_enabled
+    - zookeeper_auth_type is defined
+    - zookeeper_auth_type == 'gssapi'
   become: true
 
 - name: Copy SSL resources
   block:
-    - name: Copy java truststore file
+    - name: Copy truststore file
       ansible.builtin.copy:
         src: "{{ kafka_ssl_truststore_file }}"
         dest: "{{ kafka_ssl_truststore_location }}"
         owner: "{{ kafka_user }}"
         group: "{{ kafka_group }}"
         mode: 0400
 
-    - name: Copy java keystore file
+    - name: Copy keystore file
       ansible.builtin.copy:
         src: "{{ kafka_ssl_keystore_file }}"
         dest: "{{ kafka_ssl_keystore_location }}"
@@ -56,5 +74,5 @@
       mode: 0400
   when:
     - item.value.authentication.type is defined
-    - item.value.authentication.type in 'gssapi'
+    - item.value.authentication.type == 'gssapi'
   become: true